radix_clis/scrypto/
cmd_coverage.rs

1//! Core assumptions made in this file:
2//!
3//! 1. That all of the WASM files compiled for coverage are built using the `nightly` toolchain.
4//! 2. That the user doesn't have control over which `nightly` toolchain to use and we will just use
5//!    the `nightly` toolchain available on the system.
6//! 3. That we're always building the packages for the `wasm32-unknown-unknown` target.
7//! 4. That we're always using the `release` profile for all of the coverage builds.
8//! 5. That the user already has `clang`, `llvm-cov`, and `llvm-profdata` installed on their local
9//!    machine and available in the path.
10
11use cargo_metadata::MetadataCommand;
12use cargo_metadata::Package;
13use clap::Parser;
14use radix_engine_interface::types::Level;
15use regex::Regex;
16use sbor::prelude::*;
17use scrypto_compiler::is_scrypto_cargo_locked_env_var_active;
18use scrypto_compiler::RustFlags;
19use scrypto_compiler::ScryptoCompiler;
20use scrypto_compiler::DEFAULT_ENVIRONMENT_VARIABLES;
21use std::env::current_dir;
22use std::ffi::OsStr;
23use std::path::Path;
24use std::path::PathBuf;
25use std::process::Command;
26use std::process::Stdio;
27use std::string::FromUtf8Error;
28use std::sync::LazyLock;
29use walkdir::WalkDir;
30
31use crate::utils::*;
32
33/// Run Scrypto tests and generate code coverage report
34#[derive(Parser, Debug)]
35pub struct Coverage {
36    /// The arguments to be passed to the test executable.
37    ///
38    /// Note that these arguments will not be passed to the compilation process, only to the test
39    /// executable.
40    arguments: Vec<String>,
41
42    /// Ensures the Cargo.lock file is used as-is. Equivalent to `cargo test --locked`.
43    /// Alternatively, the `SCRYPTO_CARGO_LOCKED` environment variable can be used,
44    /// which makes it easy to set universally in CI.
45    #[clap(long)]
46    locked: bool,
47
48    /// The package directory
49    #[clap(long)]
50    path: Option<PathBuf>,
51}
52
53impl Coverage {
54    pub fn run(self) -> Result<(), CoverageError> {
55        static LLVM_IR_CORRECTIONS_REGEX: LazyLock<Regex> =
56            LazyLock::new(|| Regex::new(r"(?ms)^(define[^\n]*\n).*?^}\s*$").unwrap());
57
58        // Constructing the paths to all of the required files based on the provided package path.
59        let paths = Paths::new(self.path)?;
60
61        // Initializing the llvm-toolchain that will be used to match the current version used for
62        // the nightly compiler.
63        let llvm_toolchain = LLVMToolchain::new()?;
64
65        // Building the package with WASM instrumentation.
66        let build_environment_variables = construct_build_environment_variables();
67        ScryptoCompiler::builder()
68            .manifest_path(paths.manifest_path.as_path())
69            .log_level(Level::Trace)
70            .optimize_with_wasm_opt(None)
71            .target_directory(paths.coverage_dir_path.as_path())
72            .envs(build_environment_variables)
73            .coverage()
74            .compile()
75            .map_err(BuildError::ScryptoCompilerError)
76            .map_err(CoverageError::BuildError)?;
77
78        // Reinitializing the directories that require reinitialization.
79        paths.reinitialize_required_directories()?;
80
81        // Running the tests on the package, this will generate the profraw files for us.
82        test_package(
83            paths.package_directory_path.as_path(),
84            self.arguments.clone(),
85            true,
86            is_scrypto_cargo_locked_env_var_active() || self.locked,
87            indexmap! {
88                "COVERAGE_DIRECTORY" => paths.coverage_data_dir_path.as_path()
89            },
90        )
91        .map_err(CoverageError::TestError)?;
92
93        // Reading the LLVM-IR file of the compiled package and applying the necessary corrections
94        // to it.
95        let llvm_ir_file_pre_correction_contents =
96            std::fs::read_to_string(paths.llvm_ir_pre_corrections_file_path.as_path())?;
97        let llvm_ir_file_post_correction_contents = LLVM_IR_CORRECTIONS_REGEX
98            .replace_all(
99                llvm_ir_file_pre_correction_contents.as_str(),
100                "${1}start:\n  unreachable\n}\n",
101            )
102            .to_string();
103        std::fs::write(
104            paths.llvm_ir_post_corrections_file_path.as_path(),
105            llvm_ir_file_post_correction_contents,
106        )?;
107
108        // Converting the corrected LLVM-IR into an object file through clang.
109        let object_file_conversion_output = llvm_toolchain
110            .new_clang_command()
111            .arg(paths.llvm_ir_post_corrections_file_path.as_path())
112            .arg("-Wno-override-module")
113            .arg("-c")
114            .arg("-o")
115            .arg(paths.object_file_path.as_path())
116            .arg("--target=aarch64-unknown-linux-gnu")
117            .output()
118            .map_err(CoverageError::CommandFailedToRun)?;
119        if !object_file_conversion_output.status.success() {
120            let error = String::from_utf8_lossy(&object_file_conversion_output.stderr);
121            eprintln!("clang failed: {}", error);
122            return Err(CoverageError::ClangFailed(error.to_string()));
123        }
124
125        // Merging all of the profraw files into a profdata file.
126        let profraw_files_iterator = WalkDir::new(paths.coverage_data_dir_path.as_path())
127            .into_iter()
128            .filter_map(|entry| entry.ok())
129            .map(|entry| entry.into_path())
130            .filter(|path| {
131                path.extension()
132                    .is_some_and(|extension| extension.eq_ignore_ascii_case("profraw"))
133            });
134        let profraw_merge_output = llvm_toolchain
135            .new_llvm_profdata_command()
136            .arg("merge")
137            .arg("-sparse")
138            .args(profraw_files_iterator)
139            .arg("-o")
140            .arg(paths.profdata_file_path.as_path())
141            .output()
142            .map_err(CoverageError::CommandFailedToRun)?;
143        if !profraw_merge_output.status.success() {
144            let error = String::from_utf8_lossy(&profraw_merge_output.stderr);
145            eprintln!("clang failed: {}", error);
146            return Err(CoverageError::LlvmProfdataFailed(error.to_string()));
147        }
148
149        // Generating the final report based on the object file and the profdata that was generated.
150        let report_generation_output = llvm_toolchain
151            .new_llvm_cov_command()
152            .arg("show")
153            .arg("--instr-profile")
154            .arg(paths.profdata_file_path.as_path())
155            .arg(paths.object_file_path.as_path())
156            .arg("--show-instantiations=false")
157            .arg("--format=html")
158            .arg("--output-dir")
159            .arg(paths.report_dir_path.as_path())
160            .arg("-sources")
161            .arg(paths.package_directory_path.as_path())
162            .output()
163            .map_err(CoverageError::CommandFailedToRun)?;
164        if !report_generation_output.status.success() {
165            let error = String::from_utf8_lossy(&report_generation_output.stderr);
166            eprintln!("clang failed: {}", error);
167            return Err(CoverageError::LlvmCovFailed(error.to_string()));
168        }
169
170        Ok(())
171    }
172}
173
174/// A struct that contains all of the paths, filenames, and information on the build artifacts. Note
175/// that all of the paths contained in this struct are canonicalized and do not require the user of
176/// the struct to do it again.
177///
178/// # Note
179///
180/// You should never construct this struct yourself. You should always construct it through the
181/// [`Paths::new`] function which performs the required checks to ensure that all paths are correct.
182///
183/// This struct makes the assumption (when appropriate) that the nightly compiler is used since we
184/// always make use of it for coverage and it also assumes a release profile. There is no reason for
185/// us to turn those into arguments since this struct is EXCLUSIVELY used in this coverage module
186/// and we can safely make assumptions about how other parts of the code will act.
187#[allow(dead_code)]
188#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
189struct Paths {
190    /// The path of the package that we're doing a coverage report for.
191    pub package_directory_path: PathBuf,
192    /// The path of the `Cargo.toml` manifest of package that we're doing a coverage report for.
193    pub manifest_path: PathBuf,
194    /// The name of the package that we're doing a coverage report for. We're disallowing workspaces
195    /// at this moment of time and assuming that there will only be a single compiled WASM.
196    pub package_name: String,
197
198    /// The path of the target directory used for the compilations of all of the artifacts.
199    pub target_dir_path: PathBuf,
200    /// The path of the coverage directory within the target directory that contains everything
201    /// related to coverage.
202    pub coverage_dir_path: PathBuf,
203    /// The path of the coverage data directory within the target directory that contains everything
204    /// related to coverage.
205    pub coverage_data_dir_path: PathBuf,
206    /// The path of the directory that contains the final generated coverage report.
207    pub report_dir_path: PathBuf,
208    /// The path of the output directory that contains all of the build artifacts.
209    pub build_artifacts_dir_path: PathBuf,
210
211    /// The file name that's used for all of the artifacts built from this package. This is the same
212    /// name as the package but with the `-` replaced with `_`.
213    pub file_name: String,
214
215    /// The file name of the final WASM file that doesn't contain the schema.
216    pub wasm_file_name: String,
217    /// The file path of the final WASM file that doesn't contain the schema.
218    pub wasm_file_path: PathBuf,
219
220    /// The file name of the final WASM file that does contain the schema.
221    pub wasm_with_schema_file_name: String,
222    /// The file path of the final WASM file that does contain the schema.
223    pub wasm_with_schema_file_path: PathBuf,
224
225    /// The file name of the package definition file.
226    pub rpd_file_name: String,
227    /// The file path of the package definition file.
228    pub rpd_file_path: PathBuf,
229
230    /// The file name of the LLVM-IR file.
231    pub llvm_ir_file_name: String,
232    /// The file path of the LLVM-IR file before applying the corrections.
233    pub llvm_ir_pre_corrections_file_path: PathBuf,
234    /// The file path of the LLVM-IR file after applying the corrections.
235    pub llvm_ir_post_corrections_file_path: PathBuf,
236
237    /// The file name of the object file.
238    pub object_file_name: String,
239    /// The file path of the object file.
240    pub object_file_path: PathBuf,
241
242    /// The file name of the profdata file.
243    pub profdata_file_name: String,
244    /// The file path of the profdata file.
245    pub profdata_file_path: PathBuf,
246}
247
248impl Paths {
249    pub fn new(user_provided_path: Option<PathBuf>) -> Result<Self, CoverageError> {
250        // We either use the user provided path or the current directory path. Error out of neither
251        // path exists.
252        let package_directory_path = user_provided_path
253            .or(current_dir().ok())
254            .ok_or(CoverageError::FailedToResolvePackagePath)?
255            .canonicalize()
256            .map_err(|_| CoverageError::FailedToResolvePackagePath)?;
257        let manifest_path = assert_path_exists(package_directory_path.join("Cargo.toml"))?;
258
259        // Getting the package name through the `cargo metadata` command.
260        let metadata = MetadataCommand::new()
261            .manifest_path(manifest_path.as_path())
262            .no_deps()
263            .exec()
264            .map_err(CoverageError::CargoMetadataError)?;
265        let package_name = match metadata.packages.as_slice() {
266            [Package { name, .. }] => Ok(name.as_str()),
267            [] => Err(CoverageError::NoPackagesFound),
268            [..] => Err(CoverageError::WorkspacesNotPermitted),
269        }?
270        .to_owned();
271        let file_name = package_name.replace('-', "_");
272
273        // Creating the paths of the target directory
274        let target_dir_path = package_directory_path.join("target");
275        let coverage_dir_path = target_dir_path.join("coverage");
276        let report_dir_path = coverage_dir_path.join("report");
277        let coverage_data_dir_path = coverage_dir_path.join("data");
278        let build_artifacts_dir_path = coverage_dir_path
279            .join("wasm32-unknown-unknown")
280            .join("release");
281
282        let wasm_file_name = format!("{file_name}.wasm");
283        let wasm_file_path = build_artifacts_dir_path.join(wasm_file_name.clone());
284
285        let wasm_with_schema_file_name = format!("{file_name}_with_schema.wasm");
286        let wasm_with_schema_file_path =
287            build_artifacts_dir_path.join(wasm_with_schema_file_name.clone());
288
289        let rpd_file_name = format!("{file_name}.rpd");
290        let rpd_file_path = build_artifacts_dir_path.join(rpd_file_name.clone());
291
292        let llvm_ir_file_name = format!("{file_name}.ll");
293        let llvm_ir_pre_corrections_file_path = build_artifacts_dir_path
294            .join("deps")
295            .join(llvm_ir_file_name.clone());
296        let llvm_ir_post_corrections_file_path =
297            build_artifacts_dir_path.join(llvm_ir_file_name.clone());
298
299        let object_file_name = format!("{file_name}.o");
300        let object_file_path = build_artifacts_dir_path.join(object_file_name.clone());
301
302        let profdata_file_name = format!("{file_name}.profdata");
303        let profdata_file_path = coverage_data_dir_path.join(profdata_file_name.clone());
304
305        Ok(Self {
306            package_directory_path,
307            manifest_path,
308            package_name,
309            target_dir_path,
310            coverage_dir_path,
311            coverage_data_dir_path,
312            report_dir_path,
313            build_artifacts_dir_path,
314            file_name,
315            wasm_file_name,
316            wasm_file_path,
317            wasm_with_schema_file_name,
318            wasm_with_schema_file_path,
319            rpd_file_name,
320            rpd_file_path,
321            llvm_ir_file_name,
322            llvm_ir_pre_corrections_file_path,
323            llvm_ir_post_corrections_file_path,
324            object_file_name,
325            object_file_path,
326            profdata_file_name,
327            profdata_file_path,
328        })
329    }
330
331    /// Reinitializes any directory that requires re-initialization.
332    pub fn reinitialize_required_directories(&self) -> Result<(), CoverageError> {
333        let directory_path = self.coverage_data_dir_path.as_path();
334        let _ = std::fs::remove_dir_all(directory_path);
335        std::fs::create_dir(directory_path)?;
336        Ok(())
337    }
338}
339
340/// A struct that contains the paths and helper methods for the tools from the LLVM Toolchain that
341/// we will be using.
342///
343/// # Note
344///
345/// This struct makes the same set of assumptions made in this module.
346///
347/// Do not manually construct this struct and only construct it through the [`new`] function on the
348/// struct to perform all of the required checks.
349///
350/// [`new`]: LLVMToolchain::new
351struct LLVMToolchain {
352    /// The path of the `clang` binary used by the selected version of the rust compiler.
353    clang_path: PathBuf,
354    /// The path of the `llvm-profdata` binary used by the selected version of the rust compiler.
355    llvm_profdata_path: PathBuf,
356    /// The path of the `llvm-cov` binary used by the selected version of the rust compiler.
357    llvm_cov_path: PathBuf,
358}
359
360impl LLVMToolchain {
361    pub fn new() -> Result<Self, CoverageError> {
362        static VERSION_REGEX: LazyLock<Regex> = LazyLock::new(|| {
363            Regex::new(r"(?m)^LLVM version: (?<major>\d+)\.(?<minor>\d+)\.(?<patch>\d+)$").unwrap()
364        });
365
366        // We need to use the same version of LLVM that `rustc` is configured to use.
367        let output = new_nightly_command("rustc")
368            .arg("-vV")
369            .stdout(Stdio::piped())
370            .spawn()
371            .map_err(CoverageError::CommandFailedToRun)?
372            .wait_with_output()
373            .map_err(CoverageError::CommandFailedToRun)?;
374        let stdout_string = String::from_utf8(output.stdout)?;
375
376        // Intentional unwraps: we rely on the `rustc -vV` to output the version in a specific way
377        // and we don't have a way to recover if it doesn't produce the versions in the way that we
378        // expect.
379        let llvm_major_version = VERSION_REGEX
380            .captures(&stdout_string)
381            .expect("Can't fail")
382            .name("major")
383            .expect("Can't fail")
384            .as_str()
385            .parse::<usize>()
386            .expect("Can't fail");
387
388        Ok(Self {
389            clang_path: select_llvm_command(
390                ["clang".to_string(), format!("clang-{llvm_major_version}")],
391                llvm_major_version,
392            )?
393            .into(),
394            llvm_profdata_path: select_llvm_command(
395                [
396                    "llvm-profdata".to_string(),
397                    format!("llvm-profdata-{llvm_major_version}"),
398                ],
399                llvm_major_version,
400            )?
401            .into(),
402            llvm_cov_path: select_llvm_command(
403                [
404                    "llvm-cov".to_string(),
405                    format!("llvm-cov-{llvm_major_version}"),
406                ],
407                llvm_major_version,
408            )?
409            .into(),
410        })
411    }
412
413    /// Creates a new [`Command`] that calls `clang` at the path configured in this struct.
414    pub fn new_clang_command(&self) -> Command {
415        let mut cmd = Command::new(self.clang_path.as_path());
416        cmd.stdout(Stdio::piped()).stderr(Stdio::piped());
417        cmd
418    }
419
420    /// Creates a new [`Command`] that calls `llvm-profdata` at the path configured in this struct.
421    pub fn new_llvm_profdata_command(&self) -> Command {
422        let mut cmd = Command::new(self.llvm_profdata_path.as_path());
423        cmd.stdout(Stdio::piped()).stderr(Stdio::piped());
424        cmd
425    }
426
427    /// Creates a new [`Command`] that calls `llvm-cov` at the path configured in this struct.
428    pub fn new_llvm_cov_command(&self) -> Command {
429        let mut cmd = Command::new(self.llvm_cov_path.as_path());
430        cmd.stdout(Stdio::piped()).stderr(Stdio::piped());
431        cmd
432    }
433}
434
435/// The error type used in coverage.
436#[derive(Debug, thiserror::Error)]
437pub enum CoverageError {
438    /// We've been unable to resolve the path of the package. It could be due to the user providing
439    /// an invalid path or due to us not being able to get the current working directory or due to
440    /// us failing to canonicalize the path of the package.
441    #[error("Resolution of the package path failed.")]
442    FailedToResolvePackagePath,
443
444    /// One of the paths that are required for coverage was checked for and it doesn't exist.
445    #[error("This path doesn't exist but it must exist for the coverage tool to work: {0:?}")]
446    PathDoesntExist(PathBuf),
447
448    /// An error occurred when we tried to get the `cargo metadata` of the package.
449    #[error("Encountered an error when trying to get the cargo metadata for the package: {0}")]
450    CargoMetadataError(#[from] cargo_metadata::Error),
451
452    /// Found multiple packages when we got the metadata leading us to know that this is a workspace
453    /// which is currently not permitted.
454    #[error("The provided package is a workspace which we don't currently support")]
455    WorkspacesNotPermitted,
456
457    /// Could not find any packages when we got the `cargo metadata` and therefore there is nothing
458    /// that we can perform.
459    #[error("The provided directory doesn't contain any packages")]
460    NoPackagesFound,
461
462    /// We ran a command but it failed to run or failed during waiting.
463    #[error("Command failed to run: {0:?}")]
464    CommandFailedToRun(std::io::Error),
465
466    /// One of the commands produced stdout output that wasn't valid utf-8
467    #[error(
468        "The data the the command produced on stdout is not a valid utf-8, decoding failed: {0:?}"
469    )]
470    StdoutIsNotValidUtf8(#[from] FromUtf8Error),
471
472    /// One of the commands that we look for were not found in the system.
473    #[error("A command with the following permitted aliases was not found in the system. Is it available in $PATH?")]
474    CommandNotFound(Vec<String>),
475
476    /// An error was encountered when trying to build the package.
477    #[error("An error was encountered when trying to build the package: {0:?}")]
478    BuildError(BuildError),
479
480    /// An error was encountered when trying to test the package.
481    #[error("An error was encountered when trying to test the package: {0:?}")]
482    TestError(TestError),
483
484    /// A generic IO error.
485    #[error("An IO error was encountered: {0:?}")]
486    IoError(#[from] std::io::Error),
487
488    /// An error was encountered when running the clang command
489    #[error("An error was encountered when running the clang command: {0:?}")]
490    ClangFailed(String),
491
492    /// An error was encountered when running the llvm-profdata command
493    #[error("An error was encountered when running the llvm-profdata command: {0:?}")]
494    LlvmProfdataFailed(String),
495
496    /// An error was encountered when running the llvm-cov command
497    #[error("An error was encountered when running the llvm-cov command: {0:?}")]
498    LlvmCovFailed(String),
499}
500
501/// Check if a path exists or not. If it does then it's returned, otherwise, an error is returned.
502fn assert_path_exists<P: AsRef<Path>>(path: P) -> Result<P, CoverageError> {
503    if path.as_ref().exists() {
504        Ok(path)
505    } else {
506        Err(CoverageError::PathDoesntExist(path.as_ref().to_path_buf()))
507    }
508}
509
510/// Creates a new [`Command`] that uses the nightly compiler by setting the `RUSTUP_TOOLCHAIN`
511/// environment variable. This should be used for all of the commands that we run to ensure that we
512/// are always making use of the same compiler.
513fn new_nightly_command(program: impl AsRef<OsStr>) -> Command {
514    let mut command = Command::new(program);
515    command.env("RUSTUP_TOOLCHAIN", "nightly");
516    command
517}
518
519/// A helper method that goes through a list of llvm commands and finds the first one with
520/// the same LLVM major version. This is used to allow us to accept `clang` or `clang-21`
521/// and not force the user to have the postfixed commands installed.
522fn select_llvm_command<P: AsRef<OsStr>>(
523    commands: impl IntoIterator<Item = P> + Clone,
524    llvm_major_version: usize,
525) -> Result<P, CoverageError> {
526    let match_string = format!("version {llvm_major_version}");
527    for command in commands.clone() {
528        let Ok(output) = new_nightly_command(command.as_ref())
529            .arg("--version")
530            .stdout(Stdio::piped())
531            .spawn()
532            .map_err(CoverageError::CommandFailedToRun)
533            .and_then(|child| {
534                child
535                    .wait_with_output()
536                    .map_err(CoverageError::CommandFailedToRun)
537            })
538        else {
539            continue;
540        };
541        let Ok(stdout_string) = String::from_utf8(output.stdout) else {
542            continue;
543        };
544        if stdout_string.contains(match_string.as_str()) {
545            return Ok(command);
546        }
547    }
548    Err(CoverageError::CommandNotFound(
549        commands
550            .into_iter()
551            .map(|os_str| os_str.as_ref().to_string_lossy().to_string())
552            .collect(),
553    ))
554}
555
556/// Constructs a map of the environment variables that will be used to build the package with
557/// instrumentation.
558fn construct_build_environment_variables() -> IndexMap<String, String> {
559    let mut environment_variables = DEFAULT_ENVIRONMENT_VARIABLES
560        .clone()
561        .into_iter()
562        .flat_map(|(k, v)| v.into_set().map(|v| (k, v)))
563        .collect::<IndexMap<_, _>>();
564    let rust_flags = RustFlags::for_scrypto_compilation()
565        .with_flag("-Clto=off")
566        .with_flag("-Cinstrument-coverage")
567        .with_flag("-Zno-profiler-runtime")
568        .with_flag("--emit=llvm-ir")
569        .with_flag("-Zlocation-detail=none");
570    for (env_var, cargo_encoding) in [
571        ("RUSTFLAGS", false),
572        ("CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUSTFLAGS", false),
573        ("CARGO_ENCODED_RUSTFLAGS", true),
574    ] {
575        let encoded_rust_flags = if cargo_encoding {
576            rust_flags.encode_as_cargo_encoded_rust_flags()
577        } else {
578            rust_flags.encode_as_rust_flags()
579        };
580        environment_variables.insert(env_var.to_owned(), encoded_rust_flags);
581    }
582    environment_variables.insert("RUSTUP_TOOLCHAIN".to_owned(), "nightly".to_owned());
583    environment_variables
584}
radix_clis/scrypto/cmd_coverage.rs

radix_clis/scrypto/
cmd_coverage.rs
