ra_ap_rust_analyzer/cli/
rustc_tests.rs

1//! Run all tests in a project, similar to `cargo test`, but using the mir interpreter.
2
3use std::convert::identity;
4use std::thread::Builder;
5use std::time::{Duration, Instant};
6use std::{cell::RefCell, fs::read_to_string, panic::AssertUnwindSafe, path::PathBuf};
7
8use hir::{ChangeWithProcMacros, Crate};
9use ide::{AnalysisHost, DiagnosticCode, DiagnosticsConfig};
10use ide_db::base_db;
11use itertools::Either;
12use profile::StopWatch;
13use project_model::toolchain_info::{QueryConfig, target_data};
14use project_model::{
15    CargoConfig, ManifestPath, ProjectWorkspace, ProjectWorkspaceKind, RustLibSource,
16    RustSourceWorkspaceConfig, Sysroot,
17};
18
19use load_cargo::{LoadCargoConfig, ProcMacroServerChoice, load_workspace};
20use rustc_hash::FxHashMap;
21use vfs::{AbsPathBuf, FileId};
22use walkdir::WalkDir;
23
24use crate::cli::{Result, flags, report_metric};
25
26struct Tester {
27    host: AnalysisHost,
28    root_file: FileId,
29    pass_count: u64,
30    ignore_count: u64,
31    fail_count: u64,
32    stopwatch: StopWatch,
33}
34
35fn string_to_diagnostic_code_leaky(code: &str) -> DiagnosticCode {
36    thread_local! {
37        static LEAK_STORE: RefCell<FxHashMap<String, DiagnosticCode>> = RefCell::new(FxHashMap::default());
38    }
39    LEAK_STORE.with_borrow_mut(|s| match s.get(code) {
40        Some(c) => *c,
41        None => {
42            let v = DiagnosticCode::RustcHardError(format!("E{code}").leak());
43            s.insert(code.to_owned(), v);
44            v
45        }
46    })
47}
48
49fn detect_errors_from_rustc_stderr_file(p: PathBuf) -> FxHashMap<DiagnosticCode, usize> {
50    let text = read_to_string(p).unwrap();
51    let mut result = FxHashMap::default();
52    {
53        let mut text = &*text;
54        while let Some(p) = text.find("error[E") {
55            text = &text[p + 7..];
56            let code = string_to_diagnostic_code_leaky(&text[..4]);
57            *result.entry(code).or_insert(0) += 1;
58        }
59    }
60    result
61}
62
63impl Tester {
64    fn new() -> Result<Self> {
65        let mut path = AbsPathBuf::assert_utf8(std::env::temp_dir());
66        path.push("ra-rustc-test");
67        std::fs::create_dir_all(&path)?;
68        let tmp_file = path.join("ra-rustc-test.rs");
69        std::fs::write(&tmp_file, "")?;
70        let cargo_config = CargoConfig {
71            sysroot: Some(RustLibSource::Discover),
72            all_targets: true,
73            set_test: true,
74            ..Default::default()
75        };
76
77        let mut sysroot = Sysroot::discover(tmp_file.parent().unwrap(), &cargo_config.extra_env);
78        let loaded_sysroot =
79            sysroot.load_workspace(&RustSourceWorkspaceConfig::default_cargo(), false, &|_| ());
80        if let Some(loaded_sysroot) = loaded_sysroot {
81            sysroot.set_workspace(loaded_sysroot);
82        }
83
84        let target_data = target_data::get(
85            QueryConfig::Rustc(&sysroot, tmp_file.parent().unwrap().as_ref()),
86            None,
87            &cargo_config.extra_env,
88        );
89
90        let workspace = ProjectWorkspace {
91            kind: ProjectWorkspaceKind::DetachedFile {
92                file: ManifestPath::try_from(tmp_file).unwrap(),
93                cargo: None,
94            },
95            sysroot,
96            rustc_cfg: vec![],
97            toolchain: None,
98            target: target_data.map_err(|it| it.to_string().into()),
99            cfg_overrides: Default::default(),
100            extra_includes: vec![],
101            set_test: true,
102        };
103        let load_cargo_config = LoadCargoConfig {
104            load_out_dirs_from_check: false,
105            with_proc_macro_server: ProcMacroServerChoice::Sysroot,
106            prefill_caches: false,
107            num_worker_threads: 1,
108            proc_macro_processes: 1,
109        };
110        let (db, _vfs, _proc_macro) =
111            load_workspace(workspace, &cargo_config.extra_env, &load_cargo_config)?;
112        let host = AnalysisHost::with_database(db);
113        let db = host.raw_database();
114        let krates = Crate::all(db);
115        let root_crate = krates.iter().cloned().find(|krate| krate.origin(db).is_local()).unwrap();
116        let root_file = root_crate.root_file(db);
117        Ok(Self {
118            host,
119            root_file,
120            pass_count: 0,
121            ignore_count: 0,
122            fail_count: 0,
123            stopwatch: StopWatch::start(),
124        })
125    }
126
127    fn test(&mut self, p: PathBuf) {
128        println!("{}", p.display());
129        if p.parent().unwrap().file_name().unwrap() == "auxiliary" {
130            // These are not tests
131            return;
132        }
133        if IGNORED_TESTS.iter().any(|ig| p.file_name().is_some_and(|x| x == *ig)) {
134            println!("{p:?} IGNORE");
135            self.ignore_count += 1;
136            return;
137        }
138        let stderr_path = p.with_extension("stderr");
139        let expected = if stderr_path.exists() {
140            detect_errors_from_rustc_stderr_file(stderr_path)
141        } else {
142            FxHashMap::default()
143        };
144        let text = read_to_string(&p).unwrap();
145        let mut change = ChangeWithProcMacros::default();
146        // Ignore unstable tests, since they move too fast and we do not intend to support all of them.
147        let mut ignore_test = text.contains("#![feature");
148        // Ignore test with extern crates, as this infra don't support them yet.
149        ignore_test |= text.contains("// aux-build:") || text.contains("// aux-crate:");
150        // Ignore test with extern modules similarly.
151        ignore_test |= text.contains("mod ");
152        // These should work, but they don't, and I don't know why, so ignore them.
153        ignore_test |= text.contains("extern crate proc_macro");
154        let should_have_no_error = text.contains("// check-pass")
155            || text.contains("// build-pass")
156            || text.contains("// run-pass");
157        change.change_file(self.root_file, Some(text));
158        self.host.apply_change(change);
159        let diagnostic_config = DiagnosticsConfig::test_sample();
160
161        let res = std::thread::scope(|s| {
162            let worker = Builder::new()
163                .stack_size(40 * 1024 * 1024)
164                .spawn_scoped(s, {
165                    let diagnostic_config = &diagnostic_config;
166                    let main = std::thread::current();
167                    let analysis = self.host.analysis();
168                    let root_file = self.root_file;
169                    move || {
170                        let res = std::panic::catch_unwind(AssertUnwindSafe(move || {
171                            analysis.full_diagnostics(
172                                diagnostic_config,
173                                ide::AssistResolveStrategy::None,
174                                root_file,
175                            )
176                        }));
177                        main.unpark();
178                        res
179                    }
180                })
181                .unwrap();
182
183            let timeout = Duration::from_secs(5);
184            let now = Instant::now();
185            while now.elapsed() <= timeout && !worker.is_finished() {
186                std::thread::park_timeout(timeout - now.elapsed());
187            }
188
189            if !worker.is_finished() {
190                // attempt to cancel the worker, won't work for chalk hangs unfortunately
191                self.host.trigger_garbage_collection();
192            }
193            worker.join().and_then(identity)
194        });
195        let mut actual = FxHashMap::default();
196        let panicked = match res {
197            Err(e) => Some(Either::Left(e)),
198            Ok(Ok(diags)) => {
199                for diag in diags {
200                    if !matches!(diag.code, DiagnosticCode::RustcHardError(_)) {
201                        continue;
202                    }
203                    if !should_have_no_error && !SUPPORTED_DIAGNOSTICS.contains(&diag.code) {
204                        continue;
205                    }
206                    *actual.entry(diag.code).or_insert(0) += 1;
207                }
208                None
209            }
210            Ok(Err(e)) => Some(Either::Right(e)),
211        };
212        // Ignore tests with diagnostics that we don't emit.
213        ignore_test |= expected.keys().any(|k| !SUPPORTED_DIAGNOSTICS.contains(k));
214        if ignore_test {
215            println!("{p:?} IGNORE");
216            self.ignore_count += 1;
217        } else if let Some(panic) = panicked {
218            match panic {
219                Either::Left(panic) => {
220                    if let Some(msg) = panic
221                        .downcast_ref::<String>()
222                        .map(String::as_str)
223                        .or_else(|| panic.downcast_ref::<&str>().copied())
224                    {
225                        println!("{msg:?} ")
226                    }
227                    println!("{p:?} PANIC");
228                }
229                Either::Right(_) => println!("{p:?} CANCELLED"),
230            }
231            self.fail_count += 1;
232        } else if actual == expected {
233            println!("{p:?} PASS");
234            self.pass_count += 1;
235        } else {
236            println!("{p:?} FAIL");
237            println!("actual   (r-a)   = {actual:?}");
238            println!("expected (rustc) = {expected:?}");
239            self.fail_count += 1;
240        }
241    }
242
243    fn report(&mut self) {
244        println!(
245            "Pass count = {}, Fail count = {}, Ignore count = {}",
246            self.pass_count, self.fail_count, self.ignore_count
247        );
248        println!("Testing time and memory = {}", self.stopwatch.elapsed());
249        report_metric("rustc failed tests", self.fail_count, "#");
250        report_metric("rustc testing time", self.stopwatch.elapsed().time.as_millis() as u64, "ms");
251    }
252}
253
254/// These tests break rust-analyzer (either by panicking or hanging) so we should ignore them.
255const IGNORED_TESTS: &[&str] = &[
256    "trait-with-missing-associated-type-restriction.rs", // #15646
257    "trait-with-missing-associated-type-restriction-fixable.rs", // #15646
258    "resolve-self-in-impl.rs",
259    "basic.rs", // ../rust/tests/ui/associated-type-bounds/return-type-notation/basic.rs
260    "issue-26056.rs",
261    "float-field.rs",
262    "invalid_operator_trait.rs",
263    "type-alias-impl-trait-assoc-dyn.rs",
264    "deeply-nested_closures.rs",    // exponential time
265    "hang-on-deeply-nested-dyn.rs", // exponential time
266    "dyn-rpit-and-let.rs", // unexpected free variable with depth `^1.0` with outer binder ^0
267    "issue-16098.rs",      // Huge recursion limit for macros?
268    "issue-83471.rs", // crates/hir-ty/src/builder.rs:78:9: assertion failed: self.remaining() > 0
269];
270
271const SUPPORTED_DIAGNOSTICS: &[DiagnosticCode] = &[
272    DiagnosticCode::RustcHardError("E0023"),
273    DiagnosticCode::RustcHardError("E0046"),
274    DiagnosticCode::RustcHardError("E0063"),
275    DiagnosticCode::RustcHardError("E0107"),
276    DiagnosticCode::RustcHardError("E0117"),
277    DiagnosticCode::RustcHardError("E0133"),
278    DiagnosticCode::RustcHardError("E0210"),
279    DiagnosticCode::RustcHardError("E0268"),
280    DiagnosticCode::RustcHardError("E0308"),
281    DiagnosticCode::RustcHardError("E0384"),
282    DiagnosticCode::RustcHardError("E0407"),
283    DiagnosticCode::RustcHardError("E0432"),
284    DiagnosticCode::RustcHardError("E0451"),
285    DiagnosticCode::RustcHardError("E0507"),
286    DiagnosticCode::RustcHardError("E0583"),
287    DiagnosticCode::RustcHardError("E0559"),
288    DiagnosticCode::RustcHardError("E0616"),
289    DiagnosticCode::RustcHardError("E0618"),
290    DiagnosticCode::RustcHardError("E0624"),
291    DiagnosticCode::RustcHardError("E0774"),
292    DiagnosticCode::RustcHardError("E0767"),
293    DiagnosticCode::RustcHardError("E0777"),
294];
295
296impl flags::RustcTests {
297    pub fn run(self) -> Result<()> {
298        let mut tester = Tester::new()?;
299        let walk_dir = WalkDir::new(self.rustc_repo.join("tests/ui"));
300        eprintln!("Running tests for tests/ui");
301        for i in walk_dir {
302            let i = i?;
303            let p = i.into_path();
304            if let Some(f) = &self.filter
305                && !p.as_os_str().to_string_lossy().contains(f)
306            {
307                continue;
308            }
309            if p.extension().is_none_or(|x| x != "rs") {
310                continue;
311            }
312            if let Err(e) = std::panic::catch_unwind({
313                let tester = AssertUnwindSafe(&mut tester);
314                let p = p.clone();
315                move || {
316                    let _guard = base_db::DbPanicContext::enter(p.display().to_string());
317                    { tester }.0.test(p);
318                }
319            }) {
320                std::panic::resume_unwind(e);
321            }
322        }
323        tester.report();
324        Ok(())
325    }
326}
ra_ap_rust_analyzer/cli/rustc_tests.rs

ra_ap_rust_analyzer/cli/
rustc_tests.rs
