leo_test_framework/
lib.rs

1// Copyright (C) 2019-2026 Provable Inc.
2// This file is part of the Leo library.
3
4// The Leo library is free software: you can redistribute it and/or modify
5// it under the terms of the GNU General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8
9// The Leo library is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12// GNU General Public License for more details.
13
14// You should have received a copy of the GNU General Public License
15// along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
16
17//! This is a simple test framework for the Leo compiler.
18
19#[cfg(not(feature = "no_parallel"))]
20use rayon::prelude::*;
21
22use similar::{ChangeTag, TextDiff};
23use std::{
24    fs,
25    path::{Path, PathBuf},
26};
27use walkdir::WalkDir;
28
29enum TestFailure {
30    Panicked(String),
31    Mismatch { got: String, expected: String },
32    MissingExpectation,
33}
34
35/// Print a unified diff between expected and actual content.
36fn print_diff(expected: &str, actual: &str) {
37    let diff = TextDiff::from_lines(expected, actual);
38    let has_changes = diff.iter_all_changes().any(|c| c.tag() != ChangeTag::Equal);
39    if !has_changes {
40        return;
41    }
42    for change in diff.iter_all_changes() {
43        let sign = match change.tag() {
44            ChangeTag::Delete => "-",
45            ChangeTag::Insert => "+",
46            ChangeTag::Equal => " ",
47        };
48        eprint!("{sign}{change}");
49    }
50    eprintln!();
51}
52
53/// Pulls tests from `category`, running them through the `runner` and
54/// comparing them against expectations in previous runs.
55///
56/// The tests are `.leo` files in `tests/{category}`, and the
57/// runner receives the contents of each of them as a `&str`,
58/// returning a `String` result. A test is considered to have failed
59/// if it panics or if results differ from the previous run.
60///
61///
62/// If the environment variable `UPDATE_EXPECT` is set, no comparison to
63/// a previous result is done and the result of the current run is written
64/// to the `.out` file. If no corresponding `.out` file is found in
65/// `expectations/{category}` and `UPDATE_EXPECT` is not set, the test
66/// fails with a `MissingExpectation` error.
67pub fn run_tests(category: &str, runner: fn(&str) -> String) {
68    // This ensures error output doesn't try to display colors.
69    unsafe {
70        // SAFETY: Safety issues around `set_var` are surprisingly complicated.
71        // For now, I think marking tests as `serial` may be sufficient to
72        // address this, and in the future we'll try to think of an alternative for
73        // error output.
74        std::env::set_var("NOCOLOR", "x");
75    }
76
77    let base_tests_dir: PathBuf = [env!("CARGO_MANIFEST_DIR"), "..", "..", "tests"].iter().collect();
78
79    let base_tests_dir = base_tests_dir.canonicalize().unwrap();
80    let tests_dir = base_tests_dir.join("tests").join(category);
81    let expectations_dir = base_tests_dir.join("expectations").join(category);
82
83    let filter_string = std::env::var("TEST_FILTER").unwrap_or_default();
84    let rewrite_expectations = std::env::var("UPDATE_EXPECT").is_ok();
85
86    struct TestResult {
87        failure: Option<TestFailure>,
88        name: PathBuf,
89        wrote: bool,
90    }
91
92    let paths: Vec<PathBuf> = WalkDir::new(&tests_dir)
93        .into_iter()
94        .flatten()
95        .filter_map(|entry| {
96            let path = entry.path();
97
98            if path.to_str().is_none() {
99                panic!("Path not unicode: {}.", path.display());
100            };
101
102            let path_str = path.to_str().unwrap();
103
104            if !path_str.contains(&filter_string) || !path_str.ends_with(".leo") {
105                return None;
106            }
107
108            Some(path.into())
109        })
110        .collect();
111
112    let run_test = |path: &PathBuf| -> TestResult {
113        let contents =
114            fs::read_to_string(path).unwrap_or_else(|e| panic!("Failed to read file {}: {e}.", path.display()));
115        let result_output = std::panic::catch_unwind(|| runner(&contents));
116        if let Err(payload) = result_output {
117            let s1 = payload.downcast_ref::<&str>().map(|s| s.to_string());
118            let s2 = payload.downcast_ref::<String>().cloned();
119            let s = s1.or(s2).unwrap_or_else(|| "Unknown panic payload".to_string());
120
121            return TestResult { failure: Some(TestFailure::Panicked(s)), name: path.clone(), wrote: false };
122        }
123        let output = result_output.unwrap();
124
125        let mut expectation_path: PathBuf = expectations_dir.join(path.strip_prefix(&tests_dir).unwrap());
126        expectation_path.set_extension("out");
127
128        // It may not be ideal to the the IO below in parallel, but I'm thinking it likely won't matter.
129        if rewrite_expectations {
130            fs::write(&expectation_path, &output)
131                .unwrap_or_else(|e| panic!("Failed to write file {}: {e}.", expectation_path.display()));
132            TestResult { failure: None, name: path.clone(), wrote: true }
133        } else if !expectation_path.exists() {
134            TestResult { failure: Some(TestFailure::MissingExpectation), name: path.clone(), wrote: false }
135        } else {
136            let expected = fs::read_to_string(&expectation_path)
137                .unwrap_or_else(|e| panic!("Failed to read file {}: {e}.", expectation_path.display()));
138            if output == expected {
139                TestResult { failure: None, name: path.clone(), wrote: false }
140            } else {
141                TestResult {
142                    failure: Some(TestFailure::Mismatch { got: output, expected }),
143                    name: path.clone(),
144                    wrote: false,
145                }
146            }
147        }
148    };
149
150    #[cfg(feature = "no_parallel")]
151    let results: Vec<TestResult> = paths.iter().map(run_test).collect();
152
153    #[cfg(not(feature = "no_parallel"))]
154    let results: Vec<TestResult> = paths.par_iter().map(run_test).collect();
155
156    println!("Ran {} tests.", results.len());
157
158    let failure_count = results.iter().filter(|test_result| test_result.failure.is_some()).count();
159
160    if failure_count != 0 {
161        eprintln!("{failure_count}/{} tests failed.", results.len());
162    }
163
164    let writes = results.iter().filter(|test_result| test_result.wrote).count();
165
166    for test_result in results.iter() {
167        if let Some(test_failure) = &test_result.failure {
168            eprintln!("FAILURE: {}:", test_result.name.display());
169            match test_failure {
170                TestFailure::Panicked(s) => eprintln!("Rust panic:\n{s}"),
171                TestFailure::Mismatch { got, expected } => {
172                    eprintln!("Diff (expected -> got):");
173                    print_diff(expected, got);
174                }
175                TestFailure::MissingExpectation => {
176                    eprintln!(
177                        "Unexpected test file with no expectation. \
178                         If this test file is expected, run with UPDATE_EXPECT=1 to generate the expectation file."
179                    );
180                }
181            }
182        }
183    }
184
185    if writes != 0 {
186        println!("Wrote {}/{} expectation files for tests:", writes, results.len());
187    }
188
189    for test_result in results.iter() {
190        if test_result.wrote {
191            println!("{}", test_result.name.display());
192        }
193    }
194
195    assert!(failure_count == 0);
196}
197
198pub fn run_single_test(category: &str, path: &Path, runner: fn(&str) -> String) {
199    use std::fs;
200
201    // Canonicalize the test file path to avoid strip_prefix issues
202    let path = path.canonicalize().unwrap();
203
204    unsafe {
205        // Disable colored output in test failures
206        std::env::set_var("NOCOLOR", "x");
207    }
208
209    // Base directories
210    let base_tests_dir: PathBuf = [env!("CARGO_MANIFEST_DIR"), "..", "..", "tests"].iter().collect();
211    let base_tests_dir = base_tests_dir.canonicalize().unwrap();
212
213    let tests_dir = base_tests_dir.join("tests").join(category);
214    let expectations_dir = base_tests_dir.join("expectations").join(category);
215
216    let rewrite_expectations = std::env::var("UPDATE_EXPECT").is_ok();
217
218    // Read the test file
219    println!("Running: {}", path.display());
220    let contents = fs::read_to_string(&path).unwrap_or_else(|e| panic!("Failed to read file {}: {e}.", path.display()));
221
222    // Run the test and catch panics
223    let result_output = std::panic::catch_unwind(|| runner(&contents));
224
225    let mut wrote = false;
226
227    match result_output {
228        Err(payload) => {
229            let s1 = payload.downcast_ref::<&str>().map(|s| s.to_string());
230            let s2 = payload.downcast_ref::<String>().cloned();
231            let s = s1.or(s2).unwrap_or_else(|| "Unknown panic payload".to_string());
232
233            eprintln!("FAILURE: {}:", path.display());
234            eprintln!("Rust panic:\n{s}");
235            panic!("Test failed: {}", path.display());
236        }
237        Ok(output) => {
238            // Expectation file
239            let mut expectation_path = expectations_dir.join(path.strip_prefix(&tests_dir).unwrap());
240            expectation_path.set_extension("out");
241
242            if rewrite_expectations {
243                fs::write(&expectation_path, &output)
244                    .unwrap_or_else(|e| panic!("Failed to write file {}: {e}.", expectation_path.display()));
245                wrote = true;
246            } else if !expectation_path.exists() {
247                panic!(
248                    "Unexpected test file with no expectation at {}. \
249                     If this test file is expected, run with UPDATE_EXPECT=1 to generate the expectation file.",
250                    expectation_path.display()
251                );
252            } else {
253                let expected = fs::read_to_string(&expectation_path)
254                    .unwrap_or_else(|e| panic!("Failed to read file {}: {e}.", expectation_path.display()));
255
256                if output != expected {
257                    eprintln!("FAILURE: {}:", path.display());
258                    eprintln!("Diff (expected -> got):");
259                    print_diff(&expected, &output);
260                    panic!("Test failed: {}", path.display());
261                }
262            }
263        }
264    }
265
266    if wrote {
267        println!("Wrote expectation file for test: {}", path.display());
268    }
269}
leo_test_framework/lib.rs

leo_test_framework/
lib.rs
