wasm_bindgen_cli/

wasm_bindgen_test_runner.rs

//! A "wrapper binary" used to execute Wasm files as tests
//!
//! This binary is intended to be used as a "test runner" for Wasm binaries,
//! being compatible with `cargo test` for the Wasm target. It will
//! automatically execute `wasm-bindgen` (or the equivalent thereof) and then
//! execute either Node.js over the tests or start a server which a browser can
//! be used to run against to execute tests. In a browser mode if `CI` is in the
//! environment then it'll also attempt headless testing, spawning the server in
//! the background and then using the WebDriver protocol to execute tests.
//!
//! For more documentation about this see the `wasm-bindgen-test` crate README
//! and source code.

use anyhow::{bail, Context};
use clap::Parser;
use clap::ValueEnum;
use std::env;
use std::ffi::OsString;
use std::fs;
use std::path::PathBuf;
use std::thread;
use std::time::Duration;
use wasm_bindgen_cli_support::Bindgen;
use wasmparser::{Imports, Parser as WasmParser, Payload, TypeRef};

mod deno;
mod headless;
mod node;
mod server;
mod shell;

#[derive(Parser)]
#[command(name = "wasm-bindgen-test-runner", version, about, long_about = None)]
struct Cli {
    #[arg(
        index = 1,
        help = "The file to test. `cargo test` passes this argument for you."
    )]
    file: PathBuf,
    #[arg(long, help = "Run benchmarks")]
    bench: bool,
    #[arg(long, conflicts_with = "ignored", help = "Run ignored tests")]
    include_ignored: bool,
    #[arg(long, conflicts_with = "include_ignored", help = "Run ignored tests")]
    ignored: bool,
    #[arg(long, help = "Exactly match filters rather than by substring")]
    exact: bool,
    #[arg(
        long,
        value_name = "FILTER",
        help = "Skip tests whose names contain FILTER (this flag can be used multiple times)"
    )]
    skip: Vec<String>,
    #[arg(long, help = "List all tests and benchmarks")]
    list: bool,
    #[arg(
        long,
        help = "don't capture `console.*()` of each task, allow printing directly"
    )]
    nocapture: bool,
    #[arg(
        long,
        value_enum,
        value_name = "terse",
        help = "Configure formatting of output"
    )]
    format: Option<FormatSetting>,
    #[arg(
        index = 2,
        value_name = "FILTER",
        help = "The FILTER string is tested against the name of all tests, and only those tests \
                whose names contain the filter are run."
    )]
    filter: Option<String>,
}

impl Cli {
    fn get_args(&self, tests: &Tests) -> String {
        let include_ignored = self.include_ignored;
        let filtered = tests.filtered;

        format!(
            r#"
            // Forward runtime arguments.
            cx.include_ignored({include_ignored:?});
            cx.filtered_count({filtered});
        "#
        )
    }
}

struct Tests {
    tests: Vec<Test>,
    filtered: usize,
}

impl Tests {
    fn new() -> Self {
        Self {
            tests: Vec::new(),
            filtered: 0,
        }
    }
}

struct Test {
    // test name
    name: String,
    // symbol name
    export: String,
    ignored: bool,
}

pub fn run_cli_with_args<I, T>(args: I) -> anyhow::Result<()>
where
    I: IntoIterator<Item = T>,
    T: Into<OsString> + Clone,
{
    let cli = match Cli::try_parse_from(args) {
        Ok(a) => a,
        Err(e) => match e.kind() {
            // Passing --version and --help should not result in a failure.
            clap::error::ErrorKind::DisplayHelp | clap::error::ErrorKind::DisplayVersion => {
                print!("{e}");
                return Ok(());
            }
            _ => bail!(e),
        },
    };
    rmain(cli)
}

fn rmain(cli: Cli) -> anyhow::Result<()> {
    let mut file_name_buf = cli.file.clone();

    // Repoint the file to be read from "name.js" to "name.wasm" in the case of emscripten.
    // Rustc generates a .js and a .wasm file when targeting emscripten. It lists the .js
    // file as the primary executor which is inconsistent with what is expected here.
    if file_name_buf.extension().unwrap_or_default() == "js" {
        file_name_buf.set_extension("wasm");
    }
    let wasm = fs::read(file_name_buf).context("failed to read Wasm file")?;
    let uses_memory64 = module_uses_memory64(&wasm)?;
    let mut wasm = walrus::ModuleConfig::new()
        // Generate DWARF by default so the test runner preserves debug info for
        // ordinary wasm32 test binaries. Walrus 0.26 currently overflows while
        // remapping DWARF addresses for memory64 modules, so skip DWARF
        // generation for wasm64 inputs until upstream supports them.
        .generate_dwarf(!uses_memory64)
        .parse(&wasm)
        .context("failed to deserialize Wasm module")?;
    let mut tests = Tests::new();

    // benchmark or test
    let prefix = if cli.bench { "__wbgb_" } else { "__wbgt_" };

    'outer: for export in wasm.exports.iter() {
        let Some(name) = export.name.strip_prefix(prefix) else {
            continue;
        };
        let modifiers = name.split_once('_').expect("found invalid identifier").0;

        let Some(name) = export.name.split_once("::").map(|s| s.1) else {
            continue;
        };

        let test = Test {
            name: name.into(),
            export: export.name.clone(),
            ignored: modifiers.contains('$'),
        };

        if let Some(filter) = &cli.filter {
            let matches = if cli.exact {
                name == *filter
            } else {
                name.contains(filter)
            };

            if !matches {
                tests.filtered += 1;
                continue;
            }
        }

        for skip in &cli.skip {
            let matches = if cli.exact {
                name == *skip
            } else {
                name.contains(skip)
            };

            if matches {
                tests.filtered += 1;
                continue 'outer;
            }
        }

        if !test.ignored && cli.ignored {
            tests.filtered += 1;
        } else {
            tests.tests.push(test);
        }
    }

    if cli.list {
        for test in tests.tests {
            if cli.bench {
                println!("{}: benchmark", test.name);
            } else {
                println!("{}: test", test.name);
            }
        }

        return Ok(());
    }

    let tmpdir = tempfile::tempdir()?;

    // Support a WASM_BINDGEN_KEEP_TEST_BUILD=1 env var for debugging test files
    let tmpdir_path = if env::var("WASM_BINDGEN_KEEP_TEST_BUILD").is_ok() {
        let path = tmpdir.keep();
        println!(
            "Retaining temporary build output folder: {}",
            path.to_string_lossy()
        );
        path
    } else {
        tmpdir.path().to_path_buf()
    };

    let module = "wasm-bindgen-test";

    // Right now there's a bug where if no tests are present then the
    // `wasm-bindgen-test` runtime support isn't linked in, so just bail out
    // early saying everything is ok.
    if tests.tests.is_empty() {
        println!("no tests to run!");
        return Ok(());
    }

    // Figure out if this tests is supposed to execute in node.js or a browser.
    // That's done on a per-test-binary basis with the
    // `wasm_bindgen_test_configure` macro, which emits a custom section for us
    // to read later on.

    let custom_section = wasm.customs.remove_raw("__wasm_bindgen_test_unstable");
    let no_modules = std::env::var("WASM_BINDGEN_USE_NO_MODULE").is_ok();
    let test_mode = match custom_section {
        Some(section) if section.data.contains(&0x01) => TestMode::Browser { no_modules },
        Some(section) if section.data.contains(&0x02) => TestMode::DedicatedWorker { no_modules },
        Some(section) if section.data.contains(&0x03) => TestMode::SharedWorker { no_modules },
        Some(section) if section.data.contains(&0x04) => TestMode::ServiceWorker { no_modules },
        Some(section) if section.data.contains(&0x05) => TestMode::Node { no_modules },
        Some(section) if section.data.contains(&0x06) => TestMode::Emscripten {},
        Some(_) => bail!("invalid __wasm_bingen_test_unstable value"),
        None => {
            let mut modes = Vec::new();
            let mut add_mode =
                |mode: TestMode| std::env::var(mode.env()).is_ok().then(|| modes.push(mode));
            add_mode(TestMode::Deno);
            add_mode(TestMode::Browser { no_modules });
            add_mode(TestMode::DedicatedWorker { no_modules });
            add_mode(TestMode::SharedWorker { no_modules });
            add_mode(TestMode::ServiceWorker { no_modules });
            add_mode(TestMode::Node { no_modules });

            match modes.len() {
                0 => TestMode::Node { no_modules: true },
                1 => modes[0],
                _ => {
                    bail!(
                        "only one test mode must be set, found: `{}`",
                        modes
                            .into_iter()
                            .map(TestMode::env)
                            .collect::<Vec<_>>()
                            .join("`, `")
                    )
                }
            }
        }
    };

    let headless = env::var("NO_HEADLESS").is_err();
    let debug = env::var("WASM_BINDGEN_NO_DEBUG").is_err();

    // Gracefully handle requests to execute only node or only web tests.
    let node = matches!(test_mode, TestMode::Node { .. });

    if env::var_os("WASM_BINDGEN_TEST_ONLY_NODE").is_some() && !node {
        println!(
            "this test suite is only configured to run in a browser, \
             but we're only testing node.js tests so skipping"
        );
        return Ok(());
    }
    if env::var_os("WASM_BINDGEN_TEST_ONLY_WEB").is_some() && node {
        println!(
            "\
    This test suite is only configured to run in node.js, but we're only running
    browser tests so skipping. If you'd like to run the tests in a browser
    include this in your crate when testing:

        wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser);

    You'll likely want to put that in a `#[cfg(test)]` module or at the top of an
    integration test.\
    "
        );
        return Ok(());
    }

    let test = timeout_from_env("WASM_BINDGEN_TEST_TIMEOUT", 20);
    // Browser launch and page load follow the drivers' own default limits, but
    // never undercut a raised test timeout.
    let timeouts = headless::Timeouts {
        driver: timeout_from_env("WASM_BINDGEN_TEST_DRIVER_TIMEOUT", 5),
        startup: Duration::from_secs(60).max(test),
        page_load: Duration::from_secs(300).max(test),
        test,
    };

    let shell = shell::Shell::new();

    // Make the generated bindings available for the tests to execute against.
    shell.status("Executing bindgen...");
    let mut b = Bindgen::new();
    match test_mode {
        TestMode::Node { no_modules: true } => b.nodejs(true)?,
        TestMode::Node { no_modules: false } => b.nodejs_module(true)?,
        TestMode::Deno => b.deno(true)?,
        TestMode::Browser { .. }
        | TestMode::DedicatedWorker { .. }
        | TestMode::SharedWorker { .. }
        | TestMode::ServiceWorker { .. }
        | TestMode::Emscripten => {
            if test_mode.no_modules() {
                b.no_modules(true)?
            } else {
                b.web(true)?
            }
        }
    };

    if std::env::var("WASM_BINDGEN_SPLIT_LINKED_MODULES").is_ok() {
        b.split_linked_modules(true);
    }
    if std::env::var("WASM_BINDGEN_JS_STRING_BUILTINS").is_ok() {
        b.js_string_builtins(true);
    }
    if std::env::var("WASM_BINDGEN_KEEP_LLD_EXPORTS").is_ok() {
        b.keep_lld_exports(true);
    }

    // The path of benchmark baseline.
    let benchmark = if let Ok(path) = std::env::var("WASM_BINDGEN_BENCH_RESULT") {
        PathBuf::from(path)
    } else {
        // such as `js-sys/target/wbg_benchmark.json`
        let path = env::current_dir()
            .context("Failed to get current dir")?
            .join("target");
        // crates in the workspace that do not have a target dir.
        if cli.bench {
            fs::create_dir_all(&path)?;
        }
        path.join("wbg_benchmark.json")
    };

    // The debug here means adding some assertions and some error messages to the generated js
    // code.
    //
    // It has nothing to do with Rust.
    //
    // The tests are plain Wasm exports, reached through `__wasm`.
    b.debug(debug)
        .export_wasm(true)
        .input_module(module, wasm)
        .emit_start(false)
        .generate(&tmpdir_path)
        .context("executing `wasm-bindgen` over the Wasm file")?;
    shell.clear();

    match test_mode {
        TestMode::Node { no_modules } => {
            node::execute(module, &tmpdir_path, cli, tests, !no_modules, benchmark)?
        }
        TestMode::Deno => deno::execute(module, &tmpdir_path, cli, tests)?,
        TestMode::Emscripten => {
            let srv = server::spawn_emscripten(
                &if let Ok(address) = std::env::var("WASM_BINDGEN_TEST_ADDRESS") {
                    address.parse().unwrap()
                } else if headless {
                    "127.0.0.1:0".parse().unwrap()
                } else {
                    "127.0.0.1:8000".parse().unwrap()
                },
                &tmpdir_path,
                std::env::var("WASM_BINDGEN_TEST_NO_ORIGIN_ISOLATION").is_err(),
            )
            .context("failed to spawn server")?;
            let addr = srv.server_addr();
            if !headless {
                println!("Interactive browsers tests are now available at http://{addr}");
                println!();
                println!("Note that interactive mode is enabled because `NO_HEADLESS`");
                println!("is specified in the environment of this process. Once you're");
                println!("done with testing you'll need to kill this server with");
                println!("Ctrl-C.");
                srv.run();
                return Ok(());
            }
            println!("Tests are now available at http://{addr}");
            thread::spawn(|| srv.run());
            headless::run(&addr, &shell, &timeouts, cli.nocapture)?;
        }
        TestMode::Browser { .. }
        | TestMode::DedicatedWorker { .. }
        | TestMode::SharedWorker { .. }
        | TestMode::ServiceWorker { .. } => {
            let nocapture = cli.nocapture;
            let srv = server::spawn(
                &if let Ok(address) = std::env::var("WASM_BINDGEN_TEST_ADDRESS") {
                    address.parse().unwrap()
                } else if headless {
                    "127.0.0.1:0".parse().unwrap()
                } else {
                    "127.0.0.1:8000".parse().unwrap()
                },
                headless,
                module,
                &tmpdir_path,
                cli,
                tests,
                test_mode,
                std::env::var("WASM_BINDGEN_TEST_NO_ORIGIN_ISOLATION").is_err(),
                benchmark,
            )
            .context("failed to spawn server")?;
            let addr = srv.server_addr();

            // TODO: eventually we should provide the ability to exit at some point
            // (gracefully) here, but for now this just runs forever.
            if !headless {
                println!("Interactive browsers tests are now available at http://{addr}");
                println!();
                println!("Note that interactive mode is enabled because `NO_HEADLESS`");
                println!("is specified in the environment of this process. Once you're");
                println!("done with testing you'll need to kill this server with");
                println!("Ctrl-C.");
                srv.run();
                return Ok(());
            }

            thread::spawn(|| srv.run());
            headless::run(&addr, &shell, &timeouts, nocapture)?;
        }
    }
    Ok(())
}

fn timeout_from_env(name: &str, default_secs: u64) -> Duration {
    let Ok(value) = env::var(name) else {
        return Duration::from_secs(default_secs);
    };
    let secs = value
        .parse()
        .unwrap_or_else(|_| panic!("Could not parse '{name}'"));
    println!("Set {name} to {secs} seconds...");
    Duration::from_secs(secs)
}

fn module_uses_memory64(wasm: &[u8]) -> anyhow::Result<bool> {
    for payload in WasmParser::new(0).parse_all(wasm) {
        match payload.context("failed to inspect Wasm module")? {
            Payload::ImportSection(imports) => {
                for import in imports {
                    match import? {
                        Imports::Single(_, import) => {
                            if let TypeRef::Memory(memory) = import.ty {
                                if memory.memory64 {
                                    return Ok(true);
                                }
                            }
                        }
                        Imports::Compact1 { items, .. } => {
                            for item in items {
                                if let TypeRef::Memory(memory) = item?.ty {
                                    if memory.memory64 {
                                        return Ok(true);
                                    }
                                }
                            }
                        }
                        Imports::Compact2 { ty, .. } => {
                            if let TypeRef::Memory(memory) = ty {
                                if memory.memory64 {
                                    return Ok(true);
                                }
                            }
                        }
                    }
                }
            }
            Payload::MemorySection(memories) => {
                for memory in memories {
                    if memory?.memory64 {
                        return Ok(true);
                    }
                }
            }
            _ => {}
        }
    }

    Ok(false)
}

#[derive(Debug, Copy, Clone, Eq, PartialEq)]
enum TestMode {
    Node { no_modules: bool },
    Deno,
    Browser { no_modules: bool },
    DedicatedWorker { no_modules: bool },
    SharedWorker { no_modules: bool },
    ServiceWorker { no_modules: bool },
    Emscripten,
}

impl TestMode {
    fn is_worker(self) -> bool {
        matches!(
            self,
            Self::DedicatedWorker { .. } | Self::SharedWorker { .. } | Self::ServiceWorker { .. }
        )
    }

    fn no_modules(self) -> bool {
        match self {
            Self::Deno | Self::Emscripten => true,
            Self::Browser { no_modules }
            | Self::Node { no_modules }
            | Self::DedicatedWorker { no_modules }
            | Self::SharedWorker { no_modules }
            | Self::ServiceWorker { no_modules } => no_modules,
        }
    }

    fn env(self) -> &'static str {
        match self {
            TestMode::Node { .. } => "WASM_BINDGEN_USE_NODE_EXPERIMENTAL",
            TestMode::Deno => "WASM_BINDGEN_USE_DENO",
            TestMode::Browser { .. } => "WASM_BINDGEN_USE_BROWSER",
            TestMode::DedicatedWorker { .. } => "WASM_BINDGEN_USE_DEDICATED_WORKER",
            TestMode::SharedWorker { .. } => "WASM_BINDGEN_USE_SHARED_WORKER",
            TestMode::ServiceWorker { .. } => "WASM_BINDGEN_USE_SERVICE_WORKER",
            TestMode::Emscripten => "WASM_BINDGEN_USE_EMSCRIPTEN",
        }
    }
}

#[cfg(test)]
mod tests {
    use super::{Cli, Tests};
    use std::path::PathBuf;

    fn cli() -> Cli {
        Cli {
            file: PathBuf::from("test.wasm"),
            bench: false,
            include_ignored: true,
            ignored: false,
            exact: false,
            skip: Vec::new(),
            list: false,
            nocapture: false,
            format: None,
            filter: None,
        }
    }

    #[test]
    fn runner_args_keep_filtered_count_on_number_abi() {
        let cli = cli();
        let tests = Tests {
            tests: Vec::new(),
            filtered: 3,
        };

        let args = cli.get_args(&tests);

        assert!(args.contains("cx.include_ignored(true);"));
        assert!(args.contains("cx.filtered_count(3);"));
        assert!(!args.contains("3n"));
    }
}

/// Possible values for the `--format` option.
#[derive(Debug, Clone, Copy, ValueEnum)]
enum FormatSetting {
    /// Display one character per test
    Terse,
}