use crate::{write_file_if_changed, CargoCommandVersioned, OFFLINE};
use build_helper::rerun_if_changed;
use cargo_metadata::{CargoOpt, Metadata, MetadataCommand};
use parity_wasm::elements::{deserialize_buffer, Module};
use std::{
borrow::ToOwned,
collections::HashSet,
env, fs,
hash::{Hash, Hasher},
ops::Deref,
path::{Path, PathBuf},
process,
};
use strum::{EnumIter, IntoEnumIterator};
use toml::value::Table;
use walkdir::WalkDir;
fn colorize_info_message(message: &str) -> String {
if super::color_output_enabled() {
ansi_term::Color::Yellow.bold().paint(message).to_string()
} else {
message.into()
}
}
pub struct WasmBinaryBloaty(PathBuf);
impl WasmBinaryBloaty {
pub fn wasm_binary_bloaty_path_escaped(&self) -> String {
self.0.display().to_string().escape_default().to_string()
}
pub fn wasm_binary_bloaty_path(&self) -> &Path {
&self.0
}
}
pub struct WasmBinary(PathBuf);
impl WasmBinary {
pub fn wasm_binary_path(&self) -> &Path {
&self.0
}
pub fn wasm_binary_path_escaped(&self) -> String {
self.0.display().to_string().escape_default().to_string()
}
}
fn crate_metadata(cargo_manifest: &Path) -> Metadata {
let mut cargo_lock = cargo_manifest.to_path_buf();
cargo_lock.set_file_name("Cargo.lock");
let cargo_lock_existed = cargo_lock.exists();
let cargo_manifest = if let Some(mut cargo_lock) = find_cargo_lock(cargo_manifest) {
cargo_lock.set_file_name("Cargo.toml");
cargo_lock
} else {
cargo_manifest.to_path_buf()
};
let mut crate_metadata_command = create_metadata_command(cargo_manifest);
crate_metadata_command.features(CargoOpt::AllFeatures);
let crate_metadata = crate_metadata_command
.exec()
.expect("`cargo metadata` can not fail on project `Cargo.toml`; qed");
if !cargo_lock_existed {
let _ = fs::remove_file(&cargo_lock);
}
crate_metadata
}
pub(crate) fn create_and_compile(
project_cargo_toml: &Path,
default_rustflags: &str,
cargo_cmd: CargoCommandVersioned,
features_to_enable: Vec<String>,
wasm_binary_name: Option<String>,
check_for_runtime_version_section: bool,
) -> (Option<WasmBinary>, WasmBinaryBloaty) {
let wasm_workspace_root = get_wasm_workspace_root();
let wasm_workspace = wasm_workspace_root.join("wbuild");
let crate_metadata = crate_metadata(project_cargo_toml);
let project = create_project(
project_cargo_toml,
&wasm_workspace,
&crate_metadata,
crate_metadata.workspace_root.as_ref(),
features_to_enable,
);
let profile = build_project(&project, default_rustflags, cargo_cmd);
let (wasm_binary, wasm_binary_compressed, bloaty) =
compact_wasm_file(&project, profile, project_cargo_toml, wasm_binary_name);
if check_for_runtime_version_section {
ensure_runtime_version_wasm_section_exists(bloaty.wasm_binary_bloaty_path());
}
wasm_binary
.as_ref()
.map(|wasm_binary| copy_wasm_to_target_directory(project_cargo_toml, wasm_binary));
wasm_binary_compressed.as_ref().map(|wasm_binary_compressed| {
copy_wasm_to_target_directory(project_cargo_toml, wasm_binary_compressed)
});
let final_wasm_binary = wasm_binary_compressed.or(wasm_binary);
generate_rerun_if_changed_instructions(
project_cargo_toml,
&project,
&wasm_workspace,
final_wasm_binary.as_ref(),
&bloaty,
);
if let Err(err) = adjust_mtime(&bloaty, final_wasm_binary.as_ref()) {
build_helper::warning!("Error while adjusting the mtime of the wasm binaries: {}", err)
}
(final_wasm_binary, bloaty)
}
fn ensure_runtime_version_wasm_section_exists(wasm: &Path) {
let wasm_blob = fs::read(wasm).expect("`{wasm}` was just written and should exist; qed");
let module: Module = match deserialize_buffer(&wasm_blob) {
Ok(m) => m,
Err(e) => {
println!("Failed to deserialize `{}`: {e:?}", wasm.display());
process::exit(1);
},
};
if !module.custom_sections().any(|cs| cs.name() == "runtime_version") {
println!(
"Couldn't find the `runtime_version` wasm section. \
Please ensure that you are using the `sp_version::runtime_version` attribute macro!"
);
process::exit(1);
}
}
fn adjust_mtime(
bloaty_wasm: &WasmBinaryBloaty,
compressed_or_compact_wasm: Option<&WasmBinary>,
) -> std::io::Result<()> {
let out_dir = build_helper::out_dir();
let invoked_timestamp = out_dir.join("../invoked.timestamp");
let metadata = fs::metadata(invoked_timestamp)?;
let mtime = filetime::FileTime::from_last_modification_time(&metadata);
filetime::set_file_mtime(bloaty_wasm.wasm_binary_bloaty_path(), mtime)?;
if let Some(binary) = compressed_or_compact_wasm.as_ref() {
filetime::set_file_mtime(binary.wasm_binary_path(), mtime)?;
}
Ok(())
}
fn find_cargo_lock(cargo_manifest: &Path) -> Option<PathBuf> {
fn find_impl(mut path: PathBuf) -> Option<PathBuf> {
loop {
if path.join("Cargo.lock").exists() {
return Some(path.join("Cargo.lock"))
}
if !path.pop() {
return None
}
}
}
if let Ok(workspace) = env::var(crate::WASM_BUILD_WORKSPACE_HINT) {
let path = PathBuf::from(workspace);
if path.join("Cargo.lock").exists() {
return Some(path.join("Cargo.lock"))
} else {
build_helper::warning!(
"`{}` env variable doesn't point to a directory that contains a `Cargo.lock`.",
crate::WASM_BUILD_WORKSPACE_HINT,
);
}
}
if let Some(path) = find_impl(build_helper::out_dir()) {
return Some(path)
}
build_helper::warning!(
"Could not find `Cargo.lock` for `{}`, while searching from `{}`. \
To fix this, point the `{}` env variable to the directory of the workspace being compiled.",
cargo_manifest.display(),
build_helper::out_dir().display(),
crate::WASM_BUILD_WORKSPACE_HINT,
);
None
}
fn get_crate_name(cargo_manifest: &Path) -> String {
let cargo_toml: Table = toml::from_str(
&fs::read_to_string(cargo_manifest).expect("File exists as checked before; qed"),
)
.expect("Cargo manifest is a valid toml file; qed");
let package = cargo_toml
.get("package")
.and_then(|t| t.as_table())
.expect("`package` key exists in valid `Cargo.toml`; qed");
package
.get("name")
.and_then(|p| p.as_str())
.map(ToOwned::to_owned)
.expect("Package name exists; qed")
}
fn get_wasm_binary_name(cargo_manifest: &Path) -> String {
get_crate_name(cargo_manifest).replace('-', "_")
}
fn get_wasm_workspace_root() -> PathBuf {
let mut out_dir = build_helper::out_dir();
loop {
match out_dir.parent() {
Some(parent) if out_dir.ends_with("build") => return parent.to_path_buf(),
_ =>
if !out_dir.pop() {
break
},
}
}
panic!("Could not find target dir in: {}", build_helper::out_dir().display())
}
fn create_project_cargo_toml(
wasm_workspace: &Path,
workspace_root_path: &Path,
crate_name: &str,
crate_path: &Path,
wasm_binary: &str,
enabled_features: impl Iterator<Item = String>,
) {
let mut workspace_toml: Table = toml::from_str(
&fs::read_to_string(workspace_root_path.join("Cargo.toml"))
.expect("Workspace root `Cargo.toml` exists; qed"),
)
.expect("Workspace root `Cargo.toml` is a valid toml file; qed");
let mut wasm_workspace_toml = Table::new();
let mut release_profile = Table::new();
release_profile.insert("panic".into(), "abort".into());
release_profile.insert("lto".into(), "thin".into());
let mut production_profile = Table::new();
production_profile.insert("inherits".into(), "release".into());
production_profile.insert("lto".into(), "fat".into());
production_profile.insert("codegen-units".into(), 1.into());
let mut dev_profile = Table::new();
dev_profile.insert("panic".into(), "abort".into());
let mut profile = Table::new();
profile.insert("release".into(), release_profile.into());
profile.insert("production".into(), production_profile.into());
profile.insert("dev".into(), dev_profile.into());
wasm_workspace_toml.insert("profile".into(), profile.into());
while let Some(mut patch) =
workspace_toml.remove("patch").and_then(|p| p.try_into::<Table>().ok())
{
patch
.iter_mut()
.filter_map(|p| {
p.1.as_table_mut().map(|t| t.iter_mut().filter_map(|t| t.1.as_table_mut()))
})
.flatten()
.for_each(|p| {
p.iter_mut().filter(|(k, _)| k == &"path").for_each(|(_, v)| {
if let Some(path) = v.as_str().map(PathBuf::from) {
if path.is_relative() {
*v = workspace_root_path.join(path).display().to_string().into();
}
}
})
});
wasm_workspace_toml.insert("patch".into(), patch.into());
}
let mut package = Table::new();
package.insert("name".into(), format!("{}-wasm", crate_name).into());
package.insert("version".into(), "1.0.0".into());
package.insert("edition".into(), "2021".into());
wasm_workspace_toml.insert("package".into(), package.into());
let mut lib = Table::new();
lib.insert("name".into(), wasm_binary.into());
lib.insert("crate-type".into(), vec!["cdylib".to_string()].into());
wasm_workspace_toml.insert("lib".into(), lib.into());
let mut dependencies = Table::new();
let mut wasm_project = Table::new();
wasm_project.insert("package".into(), crate_name.into());
wasm_project.insert("path".into(), crate_path.display().to_string().into());
wasm_project.insert("default-features".into(), false.into());
wasm_project.insert("features".into(), enabled_features.collect::<Vec<_>>().into());
dependencies.insert("wasm-project".into(), wasm_project.into());
wasm_workspace_toml.insert("dependencies".into(), dependencies.into());
wasm_workspace_toml.insert("workspace".into(), Table::new().into());
write_file_if_changed(
wasm_workspace.join("Cargo.toml"),
toml::to_string_pretty(&wasm_workspace_toml).expect("Wasm workspace toml is valid; qed"),
);
}
fn find_package_by_manifest_path<'a>(
pkg_name: &str,
manifest_path: &Path,
crate_metadata: &'a cargo_metadata::Metadata,
) -> &'a cargo_metadata::Package {
if let Some(pkg) = crate_metadata.packages.iter().find(|p| p.manifest_path == manifest_path) {
return pkg
}
let pkgs_by_name = crate_metadata
.packages
.iter()
.filter(|p| p.name == pkg_name)
.collect::<Vec<_>>();
if let Some(pkg) = pkgs_by_name.first() {
if pkgs_by_name.len() > 1 {
panic!(
"Found multiple packages matching the name {pkg_name} ({manifest_path:?}): {:?}",
pkgs_by_name
);
} else {
return pkg
}
} else {
panic!("Failed to find entry for package {pkg_name} ({manifest_path:?}).");
}
}
fn project_enabled_features(
pkg_name: &str,
cargo_manifest: &Path,
crate_metadata: &cargo_metadata::Metadata,
) -> Vec<String> {
let package = find_package_by_manifest_path(pkg_name, cargo_manifest, crate_metadata);
let std_enabled = package.features.get("std");
let mut enabled_features = package
.features
.iter()
.filter(|(f, v)| {
let mut feature_env = f.replace("-", "_");
feature_env.make_ascii_uppercase();
if v.len() == 1 &&
v.get(0).map_or(false, |v| *v == format!("dep:{}", f)) &&
std_enabled.as_ref().map(|e| e.iter().any(|ef| ef == *f)).unwrap_or(false)
{
return false
}
*f != "std" &&
*f != "default" && env::var(format!("CARGO_FEATURE_{}", feature_env))
.map(|v| v == "1")
.unwrap_or_default()
})
.map(|d| d.0.clone())
.collect::<Vec<_>>();
enabled_features.sort();
enabled_features
}
fn has_runtime_wasm_feature_declared(
pkg_name: &str,
cargo_manifest: &Path,
crate_metadata: &cargo_metadata::Metadata,
) -> bool {
let package = find_package_by_manifest_path(pkg_name, cargo_manifest, crate_metadata);
package.features.keys().any(|k| k == "runtime-wasm")
}
fn create_project(
project_cargo_toml: &Path,
wasm_workspace: &Path,
crate_metadata: &Metadata,
workspace_root_path: &Path,
features_to_enable: Vec<String>,
) -> PathBuf {
let crate_name = get_crate_name(project_cargo_toml);
let crate_path = project_cargo_toml.parent().expect("Parent path exists; qed");
let wasm_binary = get_wasm_binary_name(project_cargo_toml);
let wasm_project_folder = wasm_workspace.join(&crate_name);
fs::create_dir_all(wasm_project_folder.join("src"))
.expect("Wasm project dir create can not fail; qed");
let mut enabled_features =
project_enabled_features(&crate_name, project_cargo_toml, crate_metadata);
if has_runtime_wasm_feature_declared(&crate_name, project_cargo_toml, crate_metadata) {
enabled_features.push("runtime-wasm".into());
}
let mut enabled_features = enabled_features.into_iter().collect::<HashSet<_>>();
enabled_features.extend(features_to_enable.into_iter());
create_project_cargo_toml(
&wasm_project_folder,
workspace_root_path,
&crate_name,
crate_path,
&wasm_binary,
enabled_features.into_iter(),
);
write_file_if_changed(
wasm_project_folder.join("src/lib.rs"),
"#![no_std] pub use wasm_project::*;",
);
if let Some(crate_lock_file) = find_cargo_lock(project_cargo_toml) {
crate::copy_file_if_changed(crate_lock_file, wasm_project_folder.join("Cargo.lock"));
}
wasm_project_folder
}
#[derive(Debug, EnumIter)]
enum Profile {
Debug,
Release,
Production,
}
impl Profile {
fn detect(wasm_project: &Path) -> Profile {
let (name, overriden) = if let Ok(name) = env::var(crate::WASM_BUILD_TYPE_ENV) {
(name, true)
} else {
let name = wasm_project
.components()
.rev()
.take_while(|c| c.as_os_str() != "target")
.collect::<Vec<_>>()
.iter()
.rev()
.take_while(|c| c.as_os_str() != "wbuild")
.last()
.expect("We put the wasm project within a `target/.../wbuild` path; qed")
.as_os_str()
.to_str()
.expect("All our profile directory names are ascii; qed")
.to_string();
(name, false)
};
match (Profile::iter().find(|p| p.directory() == name), overriden) {
(Some(Profile::Debug), false) => Profile::Release,
(Some(profile), _) => profile,
(None, false) => {
let profile = Profile::Release;
build_helper::warning!(
"Unknown cargo profile `{}`. Defaulted to `{:?}` for the runtime build.",
name,
profile,
);
profile
},
(None, true) => {
println!(
"Unexpected profile name: `{}`. One of the following is expected: {:?}",
name,
Profile::iter().map(|p| p.directory()).collect::<Vec<_>>(),
);
process::exit(1);
},
}
}
fn name(&self) -> &'static str {
match self {
Self::Debug => "dev",
Self::Release => "release",
Self::Production => "production",
}
}
fn directory(&self) -> &'static str {
match self {
Self::Debug => "debug",
_ => self.name(),
}
}
fn wants_compact(&self) -> bool {
!matches!(self, Self::Debug)
}
}
fn offline_build() -> bool {
env::var(OFFLINE).map_or(false, |v| v == "true")
}
fn build_project(
project: &Path,
default_rustflags: &str,
cargo_cmd: CargoCommandVersioned,
) -> Profile {
let manifest_path = project.join("Cargo.toml");
let mut build_cmd = cargo_cmd.command();
let rustflags = format!(
"-C target-cpu=mvp -C target-feature=-sign-ext -C link-arg=--export-table {} {}",
default_rustflags,
env::var(crate::WASM_BUILD_RUSTFLAGS_ENV).unwrap_or_default(),
);
build_cmd
.args(&["rustc", "--target=wasm32-unknown-unknown"])
.arg(format!("--manifest-path={}", manifest_path.display()))
.env("RUSTFLAGS", rustflags)
.env_remove("CARGO_TARGET_DIR")
.env_remove("CARGO_ENCODED_RUSTFLAGS")
.env(crate::SKIP_BUILD_ENV, "");
if super::color_output_enabled() {
build_cmd.arg("--color=always");
}
let profile = Profile::detect(project);
build_cmd.arg("--profile");
build_cmd.arg(profile.name());
if offline_build() {
build_cmd.arg("--offline");
}
println!("{}", colorize_info_message("Information that should be included in a bug report."));
println!("{} {:?}", colorize_info_message("Executing build command:"), build_cmd);
println!("{} {}", colorize_info_message("Using rustc version:"), cargo_cmd.rustc_version());
match build_cmd.status().map(|s| s.success()) {
Ok(true) => profile,
_ => process::exit(1),
}
}
fn compact_wasm_file(
project: &Path,
profile: Profile,
cargo_manifest: &Path,
out_name: Option<String>,
) -> (Option<WasmBinary>, Option<WasmBinary>, WasmBinaryBloaty) {
let default_out_name = get_wasm_binary_name(cargo_manifest);
let out_name = out_name.unwrap_or_else(|| default_out_name.clone());
let in_path = project
.join("target/wasm32-unknown-unknown")
.join(profile.directory())
.join(format!("{}.wasm", default_out_name));
let (wasm_compact_path, wasm_compact_compressed_path) = if profile.wants_compact() {
let wasm_compact_path = project.join(format!("{}.compact.wasm", out_name,));
wasm_opt::OptimizationOptions::new_opt_level_0()
.mvp_features_only()
.debug_info(true)
.add_pass(wasm_opt::Pass::StripDwarf)
.run(&in_path, &wasm_compact_path)
.expect("Failed to compact generated WASM binary.");
let wasm_compact_compressed_path =
project.join(format!("{}.compact.compressed.wasm", out_name));
if compress_wasm(&wasm_compact_path, &wasm_compact_compressed_path) {
(Some(WasmBinary(wasm_compact_path)), Some(WasmBinary(wasm_compact_compressed_path)))
} else {
(Some(WasmBinary(wasm_compact_path)), None)
}
} else {
(None, None)
};
let bloaty_path = project.join(format!("{}.wasm", out_name));
fs::copy(in_path, &bloaty_path).expect("Copying the bloaty file to the project dir.");
(wasm_compact_path, wasm_compact_compressed_path, WasmBinaryBloaty(bloaty_path))
}
fn compress_wasm(wasm_binary_path: &Path, compressed_binary_out_path: &Path) -> bool {
use sp_maybe_compressed_blob::CODE_BLOB_BOMB_LIMIT;
let data = fs::read(wasm_binary_path).expect("Failed to read WASM binary");
if let Some(compressed) = sp_maybe_compressed_blob::compress(&data, CODE_BLOB_BOMB_LIMIT) {
fs::write(compressed_binary_out_path, &compressed[..])
.expect("Failed to write WASM binary");
true
} else {
build_helper::warning!(
"Writing uncompressed wasm. Exceeded maximum size {}",
CODE_BLOB_BOMB_LIMIT,
);
false
}
}
#[derive(Debug)]
struct DeduplicatePackage<'a> {
package: &'a cargo_metadata::Package,
identifier: String,
}
impl<'a> From<&'a cargo_metadata::Package> for DeduplicatePackage<'a> {
fn from(package: &'a cargo_metadata::Package) -> Self {
Self {
package,
identifier: format!("{}{}{:?}", package.name, package.version, package.source),
}
}
}
impl<'a> Hash for DeduplicatePackage<'a> {
fn hash<H: Hasher>(&self, state: &mut H) {
self.identifier.hash(state);
}
}
impl<'a> PartialEq for DeduplicatePackage<'a> {
fn eq(&self, other: &Self) -> bool {
self.identifier == other.identifier
}
}
impl<'a> Eq for DeduplicatePackage<'a> {}
impl<'a> Deref for DeduplicatePackage<'a> {
type Target = cargo_metadata::Package;
fn deref(&self) -> &Self::Target {
self.package
}
}
fn create_metadata_command(path: impl Into<PathBuf>) -> MetadataCommand {
let mut metadata_command = MetadataCommand::new();
metadata_command.manifest_path(path);
if offline_build() {
metadata_command.other_options(vec!["--offline".to_owned()]);
}
metadata_command
}
fn generate_rerun_if_changed_instructions(
cargo_manifest: &Path,
project_folder: &Path,
wasm_workspace: &Path,
compressed_or_compact_wasm: Option<&WasmBinary>,
bloaty_wasm: &WasmBinaryBloaty,
) {
if let Some(cargo_lock) = find_cargo_lock(cargo_manifest) {
rerun_if_changed(cargo_lock);
}
let metadata = create_metadata_command(project_folder.join("Cargo.toml"))
.exec()
.expect("`cargo metadata` can not fail!");
let package = metadata
.packages
.iter()
.find(|p| p.manifest_path == cargo_manifest)
.expect("The crate package is contained in its own metadata; qed");
let mut dependencies = package.dependencies.iter().collect::<Vec<_>>();
let mut packages = HashSet::new();
packages.insert(DeduplicatePackage::from(package));
while let Some(dependency) = dependencies.pop() {
let path_or_git_dep =
dependency.source.as_ref().map(|s| s.starts_with("git+")).unwrap_or(true);
let package = metadata
.packages
.iter()
.filter(|p| !p.manifest_path.starts_with(wasm_workspace))
.find(|p| {
(path_or_git_dep || dependency.req.matches(&p.version)) && dependency.name == p.name
});
if let Some(package) = package {
if packages.insert(DeduplicatePackage::from(package)) {
dependencies.extend(package.dependencies.iter());
}
}
}
packages.iter().for_each(package_rerun_if_changed);
compressed_or_compact_wasm.map(|w| rerun_if_changed(w.wasm_binary_path()));
rerun_if_changed(bloaty_wasm.wasm_binary_bloaty_path());
println!("cargo:rerun-if-env-changed={}", crate::SKIP_BUILD_ENV);
println!("cargo:rerun-if-env-changed={}", crate::WASM_BUILD_TYPE_ENV);
println!("cargo:rerun-if-env-changed={}", crate::WASM_BUILD_RUSTFLAGS_ENV);
println!("cargo:rerun-if-env-changed={}", crate::WASM_TARGET_DIRECTORY);
println!("cargo:rerun-if-env-changed={}", crate::WASM_BUILD_TOOLCHAIN);
}
fn package_rerun_if_changed(package: &DeduplicatePackage) {
let mut manifest_path = package.manifest_path.clone();
if manifest_path.ends_with("Cargo.toml") {
manifest_path.pop();
}
WalkDir::new(&manifest_path)
.into_iter()
.filter_entry(|p| {
p.path() == manifest_path || !p.path().is_dir() || !p.path().join("Cargo.toml").exists()
})
.filter_map(|p| p.ok().map(|p| p.into_path()))
.filter(|p| {
p.is_dir() || p.extension().map(|e| e == "rs" || e == "toml").unwrap_or_default()
})
.for_each(rerun_if_changed);
}
fn copy_wasm_to_target_directory(cargo_manifest: &Path, wasm_binary: &WasmBinary) {
let target_dir = match env::var(crate::WASM_TARGET_DIRECTORY) {
Ok(path) => PathBuf::from(path),
Err(_) => return,
};
if !target_dir.is_absolute() {
println!(
"Environment variable `{}` with `{}` is not an absolute path!",
crate::WASM_TARGET_DIRECTORY,
target_dir.display(),
);
process::exit(1);
}
fs::create_dir_all(&target_dir).expect("Creates `WASM_TARGET_DIRECTORY`.");
fs::copy(
wasm_binary.wasm_binary_path(),
target_dir.join(format!("{}.wasm", get_wasm_binary_name(cargo_manifest))),
)
.expect("Copies WASM binary to `WASM_TARGET_DIRECTORY`.");
}