leo_test_framework/
lib.rs

1// Copyright (C) 2019-2025 Provable Inc.
2// This file is part of the Leo library.
3
4// The Leo library is free software: you can redistribute it and/or modify
5// it under the terms of the GNU General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8
9// The Leo library is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12// GNU General Public License for more details.
13
14// You should have received a copy of the GNU General Public License
15// along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
16
17//! This is a simple test framework for the Leo compiler.
18
19use std::{
20    collections::{HashMap, HashSet},
21    fs,
22    path::PathBuf,
23};
24use walkdir::WalkDir;
25
26enum TestFailure {
27    Panicked(String),
28    Mismatch { got: String, expected: String },
29}
30
31/// Pulls tests from `category`, running them through the `runner` and
32/// comparing them against expectations in previous runs.
33///
34/// The tests are `.leo` files in `tests/{category}`, and the
35/// runner receives the contents of each of them as a `&str`,
36/// returning a `String` result. A test is considered to have failed
37/// if it panics or if results differ from the previous run.
38///
39///
40/// If no corresponding `.out` file is found in `expecations/{category}`,
41/// or if the environment variable `REWRITE_EXPECTATIONS` is set, no
42/// comparison to a previous result is done and the result of the current
43/// run is written to the file.
44pub fn run_tests(category: &str, runner: fn(&str) -> String) {
45    // This ensures error output doesn't try to display colors.
46    unsafe {
47        // SAFETY: Safety issues around `set_var` are surprisingly complicated.
48        // For now, I think marking tests as `serial` may be sufficient to
49        // address this, and in the future we'll try to think of an alternative for
50        // error output.
51        std::env::set_var("NOCOLOR", "x");
52    }
53
54    let base_tests_dir: PathBuf = [env!("CARGO_MANIFEST_DIR"), "..", "tests"].iter().collect();
55
56    let base_tests_dir = base_tests_dir.canonicalize().unwrap();
57    let tests_dir = base_tests_dir.join("tests").join(category);
58    let expectations_dir = base_tests_dir.join("expectations").join(category);
59
60    let filter_string = std::env::var("TEST_FILTER").unwrap_or_default();
61    let rewrite_expectations = std::env::var("REWRITE_EXPECTATIONS").is_ok();
62
63    let mut test_failures = HashMap::<String, TestFailure>::new();
64    let mut test_successes = HashSet::<String>::new();
65    let mut test_writes = HashSet::<String>::new();
66
67    for entry in WalkDir::new(&tests_dir).into_iter().flatten() {
68        let path = entry.path();
69        let path_string = path.display().to_string();
70
71        if path.to_str().is_none() {
72            panic!("Path not unicode: {path_string}.");
73        };
74
75        if !path_string.contains(&filter_string) || !path_string.ends_with(".leo") {
76            continue;
77        }
78
79        let contents = fs::read_to_string(path).unwrap_or_else(|e| panic!("Failed to read file {path_string}: {e}."));
80
81        let result_output = std::panic::catch_unwind(|| runner(&contents));
82
83        if let Err(payload) = result_output {
84            let s1 = payload.downcast_ref::<&str>().map(|s| s.to_string());
85            let s2 = payload.downcast_ref::<String>().cloned();
86            let s = s1.or(s2).unwrap_or_else(|| "Unknown panic payload".to_string());
87
88            test_failures.insert(path_string, TestFailure::Panicked(s));
89            continue;
90        }
91
92        let output = result_output.unwrap();
93
94        let mut expectation_path: PathBuf = expectations_dir.join(path.strip_prefix(&tests_dir).unwrap());
95        expectation_path.set_extension("out");
96
97        if rewrite_expectations || !expectation_path.exists() {
98            fs::write(&expectation_path, &output)
99                .unwrap_or_else(|e| panic!("Failed to write file {}: {e}.", expectation_path.display()));
100            test_writes.insert(path_string);
101        } else {
102            let expected = fs::read_to_string(&expectation_path)
103                .unwrap_or_else(|e| panic!("Failed to read file {}: {e}.", expectation_path.display()));
104            if output == expected {
105                test_successes.insert(path_string);
106            } else {
107                test_failures.insert(path_string, TestFailure::Mismatch { got: output, expected });
108            }
109        }
110    }
111
112    let total_tests = test_successes.len() + test_failures.len() + test_writes.len();
113
114    println!("Ran {total_tests} tests.");
115
116    if !test_failures.is_empty() {
117        eprintln!("{}/{} tests failed.", test_failures.len(), total_tests);
118    }
119
120    for (test_path, test_failure) in test_failures.iter() {
121        eprintln!("FAILURE: {test_path}:");
122        match test_failure {
123            TestFailure::Panicked(s) => eprintln!("Rust panic:\n{s}"),
124            TestFailure::Mismatch { got, expected } => {
125                eprintln!("\ngot:\n{got}\nexpected:\n{expected}\n")
126            }
127        }
128    }
129
130    if !test_writes.is_empty() {
131        println!("Wrote {}/{} expectation files for tests:", test_writes.len(), total_tests);
132    }
133
134    for test_path in test_writes.iter() {
135        println!("{test_path}");
136    }
137
138    assert!(test_failures.is_empty());
139}
leo_test_framework/lib.rs

leo_test_framework/
lib.rs
