leo_test_framework/
lib.rs

1// Copyright (C) 2019-2026 Provable Inc.
2// This file is part of the Leo library.
3
4// The Leo library is free software: you can redistribute it and/or modify
5// it under the terms of the GNU General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8
9// The Leo library is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12// GNU General Public License for more details.
13
14// You should have received a copy of the GNU General Public License
15// along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
16
17//! This is a simple test framework for the Leo compiler.
18
19#[cfg(not(feature = "no_parallel"))]
20use rayon::prelude::*;
21
22use std::{
23    fs,
24    path::{Path, PathBuf},
25};
26use walkdir::WalkDir;
27
28enum TestFailure {
29    Panicked(String),
30    Mismatch { got: String, expected: String },
31}
32
33/// Pulls tests from `category`, running them through the `runner` and
34/// comparing them against expectations in previous runs.
35///
36/// The tests are `.leo` files in `tests/{category}`, and the
37/// runner receives the contents of each of them as a `&str`,
38/// returning a `String` result. A test is considered to have failed
39/// if it panics or if results differ from the previous run.
40///
41///
42/// If no corresponding `.out` file is found in `expecations/{category}`,
43/// or if the environment variable `REWRITE_EXPECTATIONS` is set, no
44/// comparison to a previous result is done and the result of the current
45/// run is written to the file.
46pub fn run_tests(category: &str, runner: fn(&str) -> String) {
47    // This ensures error output doesn't try to display colors.
48    unsafe {
49        // SAFETY: Safety issues around `set_var` are surprisingly complicated.
50        // For now, I think marking tests as `serial` may be sufficient to
51        // address this, and in the future we'll try to think of an alternative for
52        // error output.
53        std::env::set_var("NOCOLOR", "x");
54    }
55
56    let base_tests_dir: PathBuf = [env!("CARGO_MANIFEST_DIR"), "..", "tests"].iter().collect();
57
58    let base_tests_dir = base_tests_dir.canonicalize().unwrap();
59    let tests_dir = base_tests_dir.join("tests").join(category);
60    let expectations_dir = base_tests_dir.join("expectations").join(category);
61
62    let filter_string = std::env::var("TEST_FILTER").unwrap_or_default();
63    let rewrite_expectations = std::env::var("REWRITE_EXPECTATIONS").is_ok();
64
65    struct TestResult {
66        failure: Option<TestFailure>,
67        name: PathBuf,
68        wrote: bool,
69    }
70
71    let paths: Vec<PathBuf> = WalkDir::new(&tests_dir)
72        .into_iter()
73        .flatten()
74        .filter_map(|entry| {
75            let path = entry.path();
76
77            if path.to_str().is_none() {
78                panic!("Path not unicode: {}.", path.display());
79            };
80
81            let path_str = path.to_str().unwrap();
82
83            if !path_str.contains(&filter_string) || !path_str.ends_with(".leo") {
84                return None;
85            }
86
87            Some(path.into())
88        })
89        .collect();
90
91    let run_test = |path: &PathBuf| -> TestResult {
92        let contents =
93            fs::read_to_string(path).unwrap_or_else(|e| panic!("Failed to read file {}: {e}.", path.display()));
94        let result_output = std::panic::catch_unwind(|| runner(&contents));
95        if let Err(payload) = result_output {
96            let s1 = payload.downcast_ref::<&str>().map(|s| s.to_string());
97            let s2 = payload.downcast_ref::<String>().cloned();
98            let s = s1.or(s2).unwrap_or_else(|| "Unknown panic payload".to_string());
99
100            return TestResult { failure: Some(TestFailure::Panicked(s)), name: path.clone(), wrote: false };
101        }
102        let output = result_output.unwrap();
103
104        let mut expectation_path: PathBuf = expectations_dir.join(path.strip_prefix(&tests_dir).unwrap());
105        expectation_path.set_extension("out");
106
107        // It may not be ideal to the the IO below in parallel, but I'm thinking it likely won't matter.
108        if rewrite_expectations || !expectation_path.exists() {
109            fs::write(&expectation_path, &output)
110                .unwrap_or_else(|e| panic!("Failed to write file {}: {e}.", expectation_path.display()));
111            TestResult { failure: None, name: path.clone(), wrote: true }
112        } else {
113            let expected = fs::read_to_string(&expectation_path)
114                .unwrap_or_else(|e| panic!("Failed to read file {}: {e}.", expectation_path.display()));
115            if output == expected {
116                TestResult { failure: None, name: path.clone(), wrote: false }
117            } else {
118                TestResult {
119                    failure: Some(TestFailure::Mismatch { got: output, expected }),
120                    name: path.clone(),
121                    wrote: false,
122                }
123            }
124        }
125    };
126
127    #[cfg(feature = "no_parallel")]
128    let results: Vec<TestResult> = paths.iter().map(run_test).collect();
129
130    #[cfg(not(feature = "no_parallel"))]
131    let results: Vec<TestResult> = paths.par_iter().map(run_test).collect();
132
133    println!("Ran {} tests.", results.len());
134
135    let failure_count = results.iter().filter(|test_result| test_result.failure.is_some()).count();
136
137    if failure_count != 0 {
138        eprintln!("{failure_count}/{} tests failed.", results.len());
139    }
140
141    let writes = results.iter().filter(|test_result| test_result.wrote).count();
142
143    for test_result in results.iter() {
144        if let Some(test_failure) = &test_result.failure {
145            eprintln!("FAILURE: {}:", test_result.name.display());
146            match test_failure {
147                TestFailure::Panicked(s) => eprintln!("Rust panic:\n{s}"),
148                TestFailure::Mismatch { got, expected } => {
149                    eprintln!("\ngot:\n{got}\nexpected:\n{expected}\n")
150                }
151            }
152        }
153    }
154
155    if writes != 0 {
156        println!("Wrote {}/{} expectation files for tests:", writes, results.len());
157    }
158
159    for test_result in results.iter() {
160        if test_result.wrote {
161            println!("{}", test_result.name.display());
162        }
163    }
164
165    assert!(failure_count == 0);
166}
167
168pub fn run_single_test(category: &str, path: &Path, runner: fn(&str) -> String) {
169    use std::fs;
170
171    // Canonicalize the test file path to avoid strip_prefix issues
172    let path = path.canonicalize().unwrap();
173
174    unsafe {
175        // Disable colored output in test failures
176        std::env::set_var("NOCOLOR", "x");
177    }
178
179    // Base directories
180    let base_tests_dir: PathBuf = [env!("CARGO_MANIFEST_DIR"), "..", "tests"].iter().collect();
181    let base_tests_dir = base_tests_dir.canonicalize().unwrap();
182
183    let tests_dir = base_tests_dir.join("tests").join(category);
184    let expectations_dir = base_tests_dir.join("expectations").join(category);
185
186    let rewrite_expectations = std::env::var("REWRITE_EXPECTATIONS").is_ok();
187
188    // Read the test file
189    println!("Running: {}", path.display());
190    let contents = fs::read_to_string(&path).unwrap_or_else(|e| panic!("Failed to read file {}: {e}.", path.display()));
191
192    // Run the test and catch panics
193    let result_output = std::panic::catch_unwind(|| runner(&contents));
194
195    let mut wrote = false;
196
197    match result_output {
198        Err(payload) => {
199            let s1 = payload.downcast_ref::<&str>().map(|s| s.to_string());
200            let s2 = payload.downcast_ref::<String>().cloned();
201            let s = s1.or(s2).unwrap_or_else(|| "Unknown panic payload".to_string());
202
203            eprintln!("FAILURE: {}:", path.display());
204            eprintln!("Rust panic:\n{s}");
205            panic!("Test failed: {}", path.display());
206        }
207        Ok(output) => {
208            // Expectation file
209            let mut expectation_path = expectations_dir.join(path.strip_prefix(&tests_dir).unwrap());
210            expectation_path.set_extension("out");
211
212            if rewrite_expectations || !expectation_path.exists() {
213                fs::write(&expectation_path, &output)
214                    .unwrap_or_else(|e| panic!("Failed to write file {}: {e}.", expectation_path.display()));
215                wrote = true;
216            } else {
217                let expected = fs::read_to_string(&expectation_path)
218                    .unwrap_or_else(|e| panic!("Failed to read file {}: {e}.", expectation_path.display()));
219
220                if output != expected {
221                    eprintln!("FAILURE: {}:", path.display());
222                    eprintln!("\ngot:\n{output}\nexpected:\n{expected}\n");
223                    panic!("Test failed: {}", path.display());
224                }
225            }
226        }
227    }
228
229    if wrote {
230        println!("Wrote expectation file for test: {}", path.display());
231    }
232}
leo_test_framework/lib.rs

leo_test_framework/
lib.rs
