cargo_wasix/

lib.rs

use crate::cache::Cache;
use crate::config::Config;
use crate::utils::CommandExt;
use anyhow::{bail, Context, Result};
use std::env;
use std::fs;
use std::io;
use std::io::Read;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::time::Duration;
use tool_path::ToolPath;

mod cache;
mod config;
mod dependencies;
mod internal;
mod tool_path;
mod toolchain;
mod utils;

/// Timeout used by [`download`].
const DOWNLOAD_TIMEOUT: Duration = Duration::from_secs(30);

pub fn main() {
    // See comments in `rmain` around `*_RUNNER` for why this exists here.
    if env::var("__CARGO_WASIX_RUNNER_SHIM").is_ok() {
        let args = env::args().skip(1).collect();
        println!(
            "{}",
            serde_json::to_string(&CargoMessage::RunWithArgs { args }).unwrap(),
        );
        return;
    }

    let mut config = Config::new();
    match rmain(&mut config) {
        Ok(()) => {}
        Err(e) => {
            config.print_error(&e);
            std::process::exit(1);
        }
    }
}

#[derive(Debug)]
enum Subcommand {
    Build,
    DownloadToolchain,
    Run,
    Test,
    Bench,
    Check,
    Tree,
    Fix,
}

fn rmain(config: &mut Config) -> Result<()> {
    config.load_cache()?;

    // skip the current executable and the `wasix` inserted by Cargo
    let mut no_message_format = false;
    let mut args = env::args_os().skip(2);

    let subcommand = args.next().and_then(|s| s.into_string().ok());
    let subcommand = match subcommand.as_deref() {
        Some("build") => Subcommand::Build,
        Some("download-toolchain") => Subcommand::DownloadToolchain,
        Some("run") => Subcommand::Run,
        Some("test") => Subcommand::Test,
        Some("bench") => Subcommand::Bench,
        Some("check") => Subcommand::Check,
        Some("tree") => {
            no_message_format = true;
            Subcommand::Tree
        }
        Some("fix") => Subcommand::Fix,
        Some("self") => return internal::main(&args.collect::<Vec<_>>(), config),
        Some("version") | Some("-V") | Some("--version") => {
            let git_info = match option_env!("GIT_INFO") {
                Some(s) => format!(" ({})", s),
                None => String::new(),
            };
            println!("cargo-wasix {}{}", env!("CARGO_PKG_VERSION"), git_info);
            std::process::exit(0)
        }
        _ => print_help(),
    };

    let mut cargo = Command::new("cargo");
    cargo.arg("+wasix");
    cargo.arg(match subcommand {
        Subcommand::Build => "build",
        Subcommand::DownloadToolchain => "download-toolchain",
        Subcommand::Check => "check",
        Subcommand::Fix => "fix",
        Subcommand::Test => "test",
        Subcommand::Tree => "tree",
        Subcommand::Bench => "bench",
        Subcommand::Run => "run",
    });

    let manifest_config = if matches!(subcommand, Subcommand::DownloadToolchain) {
        Default::default()
    } else {
        read_manifest_config()?
    };

    let target = if manifest_config.dl.unwrap_or(false) {
        "wasm32-wasmer-wasi-dl"
    } else {
        "wasm32-wasmer-wasi"
    };

    cargo.arg("--target").arg(target);
    if !no_message_format {
        cargo.arg("--message-format").arg("json-render-diagnostics");
    }

    let args = args.collect::<Vec<_>>();
    for arg in args.clone() {
        if let Some(arg) = arg.to_str() {
            if arg.starts_with("--verbose") || arg.starts_with("-v") {
                config.set_verbose(true);
            }
        }

        cargo.arg(arg);
    }

    let runner_env_var = format!(
        "CARGO_TARGET_{}_RUNNER",
        target.to_uppercase().replace('-', "_")
    );

    // If Cargo actually executes a wasm file, we don't want it to. We need to
    // postprocess wasm files (wasm-opt, wasm-bindgen, etc). As a result we will
    // actually postprocess wasm files after the build. To work around this we
    // could pass `--no-run` for `test`/`bench`, but there's unfortunately no
    // equivalent for `run`. Additionally we want to learn what arguments Cargo
    // parsed to pass to each wasm file.
    //
    // To solve this all we do a bit of a switcharoo. We say that *we* are the
    // runner, and our binary is configured to simply print a json message at
    // the beginning. We'll slurp up these json messages and then actually
    // execute everything at the end.
    //
    // Also note that we check here before we actually build that a runtime is
    // present. We first check the CARGO_TARGET_WASM32_WASMER_WASI_RUNNER environement
    // variable for a user-supplied runtime (path or executable) and use the
    // default, namely `wasmer`, if it is not set.
    let (wasix_runner, using_default) = env::var(&runner_env_var)
        .map(|runner_override| (runner_override, false))
        .unwrap_or_else(|_| ("wasmer".to_string(), true));

    let mut check_deps = false;
    match subcommand {
        Subcommand::DownloadToolchain => {
            let version = args
                .first()
                .cloned()
                .map(|v| v.into_string().unwrap().into())
                .unwrap_or(toolchain::ToolchainSpec::Latest);
            let _lock = Config::acquire_lock()?;
            let chain = toolchain::install_prebuilt_toolchain(&Config::toolchain_dir()?, version)?;
            config.info(&format!(
                "Toolchain {} downloaded and installed to path {}.\nThe wasix toolchain is now ready to use.",
                chain.name,
                chain.path.display(),
            ));
            return Ok(());
        }
        Subcommand::Run | Subcommand::Bench | Subcommand::Test => {
            check_deps = true;
            if !using_default {
                // check if the override is either a valid path or command found on $PATH
                if !(Path::new(&wasix_runner).exists() || which::which(&wasix_runner).is_ok()) {
                    bail!(
                        "failed to find `{}` (specified by ${runner_env_var}) \
                         on the filesytem or in $PATH, you'll want to fix the path or unset \
                         the ${runner_env_var} environment variable before \
                         running this command\n",
                        &wasix_runner
                    );
                }
            } else if which::which(&wasix_runner).is_err() {
                let mut msg = format!(
                    "failed to find `{}` in $PATH, you'll want to \
                     install `{}` before running this command\n",
                    wasix_runner, wasix_runner
                );
                // Because we know what runtime is being used here, we can print
                // out installation information.
                msg.push_str("you can also install through a shell:\n\n");
                msg.push_str("\tcurl https://get.wasmer.io -sSfL | sh\n");
                bail!("{}", msg);
            }
            cargo.env("__CARGO_WASIX_RUNNER_SHIM", "1");
            cargo.env(runner_env_var, env::current_exe()?);
        }
        Subcommand::Build | Subcommand::Check => check_deps = true,
        Subcommand::Tree | Subcommand::Fix => {}
    }

    let update_check_opt = if config.is_offline {
        Some(internal::UpdateCheck::new(config))
    } else {
        None
    };
    let toolchain = toolchain::ensure_toolchain(config)?;

    std::env::set_var("RUSTUP_TOOLCHAIN", &toolchain.name);

    // Set some flags for rustc (only if RUSTFLAGS is not already set)
    if std::env::var("RUSTFLAGS").is_err() {
        env::set_var("RUSTFLAGS", "-C target-feature=+atomics");
    }

    // Check the dependencies, if needed, before running cargo.
    if check_deps {
        if let Err(err) = dependencies::check(config, target) {
            config.warn(&format!("failed to check dependencies: {err}"));
        }
    }

    // Run the cargo commands
    let build = execute_cargo(&mut cargo, config, manifest_config)?;

    config.info("Post-processing WebAssembly files");

    for (wasm, profile, fresh) in build.wasms.iter() {
        // Cargo will always overwrite our `wasm` above with its own internal
        // cache. It's internal cache largely uses hard links.
        //
        // If `fresh` is *false*, then Cargo just built `wasm` and we need to
        // process it. If `fresh` is *true*, then we may have previously
        // processed it. If our previous processing was successful the output
        // was placed at `*.wasi.wasm`, so we use that to overwrite the
        // `*.wasm` file. In the process we also create a `*.rustc.wasm` for
        // debugging.
        //
        // Note that we remove files before renaming and such to ensure that
        // we're not accidentally updating the wrong hard link and such.
        let temporary_rustc = wasm.with_extension("rustc.wasm");
        let temporary_wasi = wasm.with_extension("wasi.wasm");

        drop(fs::remove_file(&temporary_rustc));
        fs::rename(wasm, &temporary_rustc)?;
        if !*fresh || !temporary_wasi.exists() {
            let result = process_wasm(&temporary_wasi, &temporary_rustc, profile, &build, config);
            result.with_context(|| {
                format!("failed to process wasm at `{}`", temporary_rustc.display())
            })?;
        }
        drop(fs::remove_file(wasm));
        fs::hard_link(&temporary_wasi, wasm)
            .or_else(|_| fs::copy(&temporary_wasi, wasm).map(|_| ()))?;
    }

    for run in build.runs.iter() {
        config.status("Running", &format!("`{}`", run.join(" ")));
        let mut cmd = Command::new(&wasix_runner);

        if wasix_runner == "wasmer" {
            cmd.arg("--enable-threads");
        }

        cmd.arg("--")
            .args(run.iter())
            .run()
            .map_err(|e| utils::hide_normal_process_exit(e, config))?;
    }

    if let Some(check) = update_check_opt {
        check.print();
    }
    Ok(())
}

pub const HELP: &str = include_str!("txt/help.txt");

fn print_help() -> ! {
    println!("{}", HELP);
    std::process::exit(0)
}

#[derive(Default, Debug)]
struct CargoBuild {
    // The version of `wasm-bindgen` used in this build, if any.
    wasm_bindgen: Option<String>,
    // The `*.wasm` artifacts we found during this build, in addition to the
    // profile that they were built with and whether or not it was `fresh`
    // during this build.
    wasms: Vec<(PathBuf, Profile, bool)>,
    // executed commands as part of the cargo build
    runs: Vec<Vec<String>>,
    // Configuration we found in the `Cargo.toml` workspace manifest for these
    // builds.
    manifest_config: ManifestConfig,
}

#[derive(serde::Deserialize, serde::Serialize, Debug, Clone)]
struct Profile {
    opt_level: String,
    debuginfo: Option<u32>,
    test: bool,
}

#[derive(serde::Deserialize, Debug, Default)]
#[serde(rename_all = "kebab-case")]
struct ManifestConfig {
    dl: Option<bool>,
    wasm_opt: Option<bool>,
    wasm_name_section: Option<bool>,
    wasm_producers_section: Option<bool>,
}

#[derive(serde::Deserialize)]
struct CargoMetadata {
    workspace_root: String,
}

#[derive(serde::Deserialize, Debug)]
struct CargoManifest {
    package: Option<CargoPackage>,
}

#[derive(serde::Deserialize, Debug)]
struct CargoPackage {
    metadata: Option<ManifestConfig>,
}

#[derive(serde::Deserialize, serde::Serialize)]
#[serde(tag = "reason", rename_all = "kebab-case")]
enum CargoMessage {
    CompilerArtifact {
        filenames: Vec<String>,
        package_id: String,
        profile: Profile,
        fresh: bool,
    },
    BuildScriptExecuted,
    RunWithArgs {
        args: Vec<String>,
    },
    BuildFinished,
}

impl CargoBuild {
    fn enable_name_section(&self, profile: &Profile) -> bool {
        match profile.debuginfo {
            Some(0) | None => self.manifest_config.wasm_name_section.unwrap_or(true),
            Some(_) => true,
        }
    }

    fn enable_producers_section(&self, profile: &Profile) -> bool {
        match profile.debuginfo {
            Some(0) | None => self.manifest_config.wasm_producers_section.unwrap_or(true),
            Some(_) => true,
        }
    }
}

/// Process a wasm file that doesn't use `wasm-bindgen`.
fn process_wasm(
    wasm: &Path,
    temp: &Path,
    profile: &Profile,
    build: &CargoBuild,
    config: &Config,
) -> Result<()> {
    config.verbose(|| {
        config.status("Processing", &temp.display().to_string());
    });
    run_wasm_opt(wasm, temp, profile, build, config)?;
    Ok(())
}

fn run_wasm_opt(
    wasm: &Path,
    temp: &Path,
    profile: &Profile,
    build: &CargoBuild,
    config: &Config,
) -> Result<()> {
    // Allow explicitly disabling wasm-opt via `Cargo.toml`.
    if build.manifest_config.wasm_opt == Some(false) {
        std::fs::rename(temp, wasm).context("failed to rename build output")?;
        return Ok(());
    }

    config.status("Optimizing", "with wasm-opt");
    let wasm_opt = config.get_wasm_opt();

    let mut cmd = Command::new(wasm_opt.bin_path());
    cmd.arg(temp);
    cmd.arg(format!("-O{}", profile.opt_level));
    cmd.arg("-o").arg(wasm);
    cmd.arg("--enable-bulk-memory");
    cmd.arg("--enable-threads");
    cmd.arg("--enable-reference-types");
    cmd.arg("--no-validation");
    cmd.arg("--translate-to-exnref");

    if !build.enable_producers_section(profile) {
        cmd.arg("--strip-producers");
    }

    match profile.debuginfo {
        Some(0) | None => {
            // release build
            if build.enable_name_section(profile) {
                cmd.arg("--debuginfo");
            } else {
                cmd.arg("--strip-debug");
            }
        }
        Some(_) if profile.opt_level == "0" => {
            // debug build
            cmd.arg("--debuginfo");
        }
        _ => {
            // release build
            cmd.arg("--strip-debug");
        }
    }

    run_or_download(
        wasm_opt.bin_path(),
        wasm_opt.is_overridden(),
        &mut cmd,
        config,
        || install_wasm_opt(&wasm_opt, config),
    )
    .context("`wasm-opt` failed to execute")?;
    Ok(())
}

fn read_manifest_config() -> Result<ManifestConfig> {
    let output = Command::new("cargo")
        .arg("metadata")
        .arg("--no-deps")
        .arg("--format-version=1")
        .capture_stdout()?;
    let metadata = serde_json::from_str::<CargoMetadata>(&output)
        .context("failed to deserialize `cargo metadata`")?;

    let manifest = Path::new(&metadata.workspace_root).join("Cargo.toml");
    let toml = fs::read_to_string(&manifest)
        .context(format!("failed to read manifest: {}", manifest.display()))?;
    let toml = toml::from_str::<CargoManifest>(&toml).context(format!(
        "failed to deserialize as TOML: {}",
        manifest.display()
    ))?;

    if let Some(meta) = toml.package.and_then(|p| p.metadata) {
        Ok(meta)
    } else {
        Ok(ManifestConfig::default())
    }
}

/// Executes the `cargo` command, reading all of the JSON that pops out and
/// parsing that into a `CargoBuild`.
fn execute_cargo(
    cargo: &mut Command,
    config: &Config,
    manifest_config: ManifestConfig,
) -> Result<CargoBuild> {
    config.verbose(|| config.status("Running", &format!("{:?}", cargo)));
    let mut process = cargo
        .stdout(Stdio::piped())
        .spawn()
        .context("failed to spawn `cargo`")?;
    let mut json = String::new();
    process
        .stdout
        .take()
        .unwrap()
        .read_to_string(&mut json)
        .context("failed to read cargo stdout into a json string")?;
    let status = process.wait().context("failed to wait on `cargo`")?;
    utils::check_success(cargo, &status, &[], &[])
        .map_err(|e| utils::hide_normal_process_exit(e, config))?;

    let mut build = CargoBuild::default();

    for line in json.lines() {
        if !line.starts_with('{') {
            println!("{}", line);
            continue;
        }
        match serde_json::from_str(line) {
            Ok(CargoMessage::CompilerArtifact {
                filenames,
                profile,
                package_id,
                fresh,
            }) => {
                let mut parts = package_id.split_whitespace();
                if parts.next() == Some("wasm-bindgen") {
                    if let Some(version) = parts.next() {
                        build.wasm_bindgen = Some(version.to_string());
                    }
                }
                for file in filenames {
                    let file = PathBuf::from(file);
                    if file.extension().and_then(|s| s.to_str()) == Some("wasm") {
                        build.wasms.push((file, profile.clone(), fresh));
                    }
                }
            }
            Ok(CargoMessage::RunWithArgs { args }) => build.runs.push(args),
            Ok(CargoMessage::BuildScriptExecuted) => {}
            Ok(CargoMessage::BuildFinished) => {}
            Err(e) => bail!("failed to parse {}: {}", line, e),
        }
    }

    build.manifest_config = manifest_config;

    Ok(build)
}

/// Attempts to execute `cmd` which is executing `requested`.
///
/// If the execution fails because `requested` isn't found *and* `requested` is
/// the same as the `cache` path provided, then `download` is invoked to
/// download the tool and then we re-execute `cmd` after the download has
/// finished.
///
/// Additionally nice diagnostics and such are printed along the way.
fn run_or_download(
    requested: &Path,
    is_overridden: bool,
    cmd: &mut Command,
    config: &Config,
    download: impl FnOnce() -> Result<()>,
) -> Result<()> {
    // NB: this is explicitly set up so that, by default, we simply execute the
    // command and assume that it exists. That should ideally avoid a few extra
    // syscalls to detect "will things work?"
    config.verbose(|| {
        if requested.exists() {
            config.status("Running", &format!("{:?}", cmd));
        }
    });

    let err = match cmd.run() {
        Ok(()) => return Ok(()),
        Err(e) => e,
    };
    let rerun_after_download = err.chain().any(|e| {
        // NotFound means we need to clearly download, PermissionDenied may mean
        // that we were racing a download and the file wasn't executable, so
        // fall through and wait for the download to finish to try again.
        if let Some(err) = e.downcast_ref::<io::Error>() {
            return err.kind() == io::ErrorKind::NotFound
                || err.kind() == io::ErrorKind::PermissionDenied;
        }
        false
    });

    // This may have failed for some reason other than `NotFound`, in which case
    // it's a legitimate error. Additionally `requested` may not actually be a
    // path that we download, in which case there's also nothing that we can do.
    if !rerun_after_download || is_overridden {
        return Err(err);
    }

    download()?;
    config.verbose(|| {
        config.status("Running", &format!("{:?}", cmd));
    });
    cmd.run()
}

fn install_wasm_opt(path: &ToolPath, config: &Config) -> Result<()> {
    let tag = "version_123";
    let binaryen_url = |target: &str| {
        let mut url = "https://github.com/WebAssembly/binaryen/releases/download/".to_string();
        url.push_str(tag);
        url.push_str("/binaryen-");
        url.push_str(tag);
        url.push('-');
        url.push_str(target);
        url.push_str(".tar.gz");
        url
    };

    let url = if cfg!(target_os = "linux") && cfg!(target_arch = "x86_64") {
        binaryen_url("x86_64-linux")
    } else if cfg!(target_os = "macos") && cfg!(target_arch = "x86_64") {
        binaryen_url("x86_64-macos")
    } else if cfg!(target_os = "macos") && cfg!(target_arch = "aarch64") {
        binaryen_url("arm64-macos")
    } else if cfg!(target_os = "windows") && cfg!(target_arch = "x86_64") {
        binaryen_url("x86_64-windows")
    } else {
        bail!(
            "no precompiled binaries of `wasm-opt` are available for this \
             platform, you'll want to set `$WASM_OPT` to a preinstalled \
             `wasm-opt` command or disable via `wasm-opt = false` in \
             your manifest"
        )
    };

    let (base_path, sub_paths) = path.cache_paths().unwrap();
    download(
        &url,
        &format!("precompiled wasm-opt {}", tag),
        base_path,
        sub_paths,
        config,
    )
}

fn download(
    url: &str,
    name: &str,
    parent: &Path,
    sub_paths: &Vec<PathBuf>,
    config: &Config,
) -> Result<()> {
    // Globally lock ourselves downloading things to coordinate with any other
    // instances of `cargo-wasi` doing a download. This is a bit coarse, but it
    // gets the job done. Additionally if someone else does the download for us
    // then we can simply return.
    let _flock = utils::flock(&config.cache().root().join("downloading"));
    if sub_paths
        .iter()
        .all(|sub_path| parent.join(sub_path).exists())
    {
        return Ok(());
    }

    // Ok, let's actually do the download
    config.status("Downloading", name);
    config.verbose(|| config.status("Get", url));

    let response = utils::get(url, DOWNLOAD_TIMEOUT)?;
    (|| -> Result<()> {
        fs::create_dir_all(parent)
            .context(format!("failed to create directory `{}`", parent.display()))?;

        let decompressed = flate2::read::GzDecoder::new(response);
        let mut tar = tar::Archive::new(decompressed);
        for entry in tar.entries()? {
            let mut entry = entry?;
            let path = entry.path()?.into_owned();
            for sub_path in sub_paths {
                if path.ends_with(sub_path) {
                    let entry_path = parent.join(sub_path);
                    let dir = entry_path.parent().unwrap();
                    if !dir.exists() {
                        fs::create_dir_all(dir)
                            .context(format!("failed to create directory `{}`", dir.display()))?;
                    }
                    entry.unpack(entry_path)?;
                }
            }
        }

        if let Some(missing) = sub_paths
            .iter()
            .find(|sub_path| !parent.join(sub_path).exists())
        {
            bail!("failed to find {:?} in archive", missing);
        }
        Ok(())
    })()
    .context(format!("failed to extract tarball from {}", url))
}