ra_ap_rust_analyzer/cli/
rustc_tests.rs

1//! Run all tests in a project, similar to `cargo test`, but using the mir interpreter.
2
3use std::convert::identity;
4use std::thread::Builder;
5use std::time::{Duration, Instant};
6use std::{cell::RefCell, fs::read_to_string, panic::AssertUnwindSafe, path::PathBuf};
7
8use hir::{ChangeWithProcMacros, Crate};
9use ide::{AnalysisHost, DiagnosticCode, DiagnosticsConfig};
10use ide_db::base_db;
11use itertools::Either;
12use profile::StopWatch;
13use project_model::toolchain_info::{QueryConfig, target_data};
14use project_model::{
15    CargoConfig, ManifestPath, ProjectWorkspace, ProjectWorkspaceKind, RustLibSource,
16    RustSourceWorkspaceConfig, Sysroot,
17};
18
19use load_cargo::{LoadCargoConfig, ProcMacroServerChoice, load_workspace};
20use rustc_hash::FxHashMap;
21use vfs::{AbsPathBuf, FileId};
22use walkdir::WalkDir;
23
24use crate::cli::{Result, flags, report_metric};
25
26struct Tester {
27    host: AnalysisHost,
28    root_file: FileId,
29    pass_count: u64,
30    ignore_count: u64,
31    fail_count: u64,
32    stopwatch: StopWatch,
33}
34
35fn string_to_diagnostic_code_leaky(code: &str) -> DiagnosticCode {
36    thread_local! {
37        static LEAK_STORE: RefCell<FxHashMap<String, DiagnosticCode>> = RefCell::new(FxHashMap::default());
38    }
39    LEAK_STORE.with_borrow_mut(|s| match s.get(code) {
40        Some(c) => *c,
41        None => {
42            let v = DiagnosticCode::RustcHardError(format!("E{code}").leak());
43            s.insert(code.to_owned(), v);
44            v
45        }
46    })
47}
48
49fn detect_errors_from_rustc_stderr_file(p: PathBuf) -> FxHashMap<DiagnosticCode, usize> {
50    let text = read_to_string(p).unwrap();
51    let mut result = FxHashMap::default();
52    {
53        let mut text = &*text;
54        while let Some(p) = text.find("error[E") {
55            text = &text[p + 7..];
56            let code = string_to_diagnostic_code_leaky(&text[..4]);
57            *result.entry(code).or_insert(0) += 1;
58        }
59    }
60    result
61}
62
63impl Tester {
64    fn new() -> Result<Self> {
65        let mut path = AbsPathBuf::assert_utf8(std::env::temp_dir());
66        path.push("ra-rustc-test");
67        let tmp_file = path.join("ra-rustc-test.rs");
68        std::fs::write(&tmp_file, "")?;
69        let cargo_config = CargoConfig {
70            sysroot: Some(RustLibSource::Discover),
71            all_targets: true,
72            set_test: true,
73            ..Default::default()
74        };
75
76        let mut sysroot = Sysroot::discover(tmp_file.parent().unwrap(), &cargo_config.extra_env);
77        let loaded_sysroot =
78            sysroot.load_workspace(&RustSourceWorkspaceConfig::default_cargo(), false, &|_| ());
79        if let Some(loaded_sysroot) = loaded_sysroot {
80            sysroot.set_workspace(loaded_sysroot);
81        }
82
83        let target_data = target_data::get(
84            QueryConfig::Rustc(&sysroot, tmp_file.parent().unwrap().as_ref()),
85            None,
86            &cargo_config.extra_env,
87        );
88
89        let workspace = ProjectWorkspace {
90            kind: ProjectWorkspaceKind::DetachedFile {
91                file: ManifestPath::try_from(tmp_file).unwrap(),
92                cargo: None,
93            },
94            sysroot,
95            rustc_cfg: vec![],
96            toolchain: None,
97            target: target_data.map_err(|it| it.to_string().into()),
98            cfg_overrides: Default::default(),
99            extra_includes: vec![],
100            set_test: true,
101        };
102        let load_cargo_config = LoadCargoConfig {
103            load_out_dirs_from_check: false,
104            with_proc_macro_server: ProcMacroServerChoice::Sysroot,
105            prefill_caches: false,
106            proc_macro_processes: 1,
107        };
108        let (db, _vfs, _proc_macro) =
109            load_workspace(workspace, &cargo_config.extra_env, &load_cargo_config)?;
110        let host = AnalysisHost::with_database(db);
111        let db = host.raw_database();
112        let krates = Crate::all(db);
113        let root_crate = krates.iter().cloned().find(|krate| krate.origin(db).is_local()).unwrap();
114        let root_file = root_crate.root_file(db);
115        Ok(Self {
116            host,
117            root_file,
118            pass_count: 0,
119            ignore_count: 0,
120            fail_count: 0,
121            stopwatch: StopWatch::start(),
122        })
123    }
124
125    fn test(&mut self, p: PathBuf) {
126        println!("{}", p.display());
127        if p.parent().unwrap().file_name().unwrap() == "auxiliary" {
128            // These are not tests
129            return;
130        }
131        if IGNORED_TESTS.iter().any(|ig| p.file_name().is_some_and(|x| x == *ig)) {
132            println!("{p:?} IGNORE");
133            self.ignore_count += 1;
134            return;
135        }
136        let stderr_path = p.with_extension("stderr");
137        let expected = if stderr_path.exists() {
138            detect_errors_from_rustc_stderr_file(stderr_path)
139        } else {
140            FxHashMap::default()
141        };
142        let text = read_to_string(&p).unwrap();
143        let mut change = ChangeWithProcMacros::default();
144        // Ignore unstable tests, since they move too fast and we do not intend to support all of them.
145        let mut ignore_test = text.contains("#![feature");
146        // Ignore test with extern crates, as this infra don't support them yet.
147        ignore_test |= text.contains("// aux-build:") || text.contains("// aux-crate:");
148        // Ignore test with extern modules similarly.
149        ignore_test |= text.contains("mod ");
150        // These should work, but they don't, and I don't know why, so ignore them.
151        ignore_test |= text.contains("extern crate proc_macro");
152        let should_have_no_error = text.contains("// check-pass")
153            || text.contains("// build-pass")
154            || text.contains("// run-pass");
155        change.change_file(self.root_file, Some(text));
156        self.host.apply_change(change);
157        let diagnostic_config = DiagnosticsConfig::test_sample();
158
159        let res = std::thread::scope(|s| {
160            let worker = Builder::new()
161                .stack_size(40 * 1024 * 1024)
162                .spawn_scoped(s, {
163                    let diagnostic_config = &diagnostic_config;
164                    let main = std::thread::current();
165                    let analysis = self.host.analysis();
166                    let root_file = self.root_file;
167                    move || {
168                        let res = std::panic::catch_unwind(AssertUnwindSafe(move || {
169                            analysis.full_diagnostics(
170                                diagnostic_config,
171                                ide::AssistResolveStrategy::None,
172                                root_file,
173                            )
174                        }));
175                        main.unpark();
176                        res
177                    }
178                })
179                .unwrap();
180
181            let timeout = Duration::from_secs(5);
182            let now = Instant::now();
183            while now.elapsed() <= timeout && !worker.is_finished() {
184                std::thread::park_timeout(timeout - now.elapsed());
185            }
186
187            if !worker.is_finished() {
188                // attempt to cancel the worker, won't work for chalk hangs unfortunately
189                self.host.trigger_garbage_collection();
190            }
191            worker.join().and_then(identity)
192        });
193        let mut actual = FxHashMap::default();
194        let panicked = match res {
195            Err(e) => Some(Either::Left(e)),
196            Ok(Ok(diags)) => {
197                for diag in diags {
198                    if !matches!(diag.code, DiagnosticCode::RustcHardError(_)) {
199                        continue;
200                    }
201                    if !should_have_no_error && !SUPPORTED_DIAGNOSTICS.contains(&diag.code) {
202                        continue;
203                    }
204                    *actual.entry(diag.code).or_insert(0) += 1;
205                }
206                None
207            }
208            Ok(Err(e)) => Some(Either::Right(e)),
209        };
210        // Ignore tests with diagnostics that we don't emit.
211        ignore_test |= expected.keys().any(|k| !SUPPORTED_DIAGNOSTICS.contains(k));
212        if ignore_test {
213            println!("{p:?} IGNORE");
214            self.ignore_count += 1;
215        } else if let Some(panic) = panicked {
216            match panic {
217                Either::Left(panic) => {
218                    if let Some(msg) = panic
219                        .downcast_ref::<String>()
220                        .map(String::as_str)
221                        .or_else(|| panic.downcast_ref::<&str>().copied())
222                    {
223                        println!("{msg:?} ")
224                    }
225                    println!("{p:?} PANIC");
226                }
227                Either::Right(_) => println!("{p:?} CANCELLED"),
228            }
229            self.fail_count += 1;
230        } else if actual == expected {
231            println!("{p:?} PASS");
232            self.pass_count += 1;
233        } else {
234            println!("{p:?} FAIL");
235            println!("actual   (r-a)   = {actual:?}");
236            println!("expected (rustc) = {expected:?}");
237            self.fail_count += 1;
238        }
239    }
240
241    fn report(&mut self) {
242        println!(
243            "Pass count = {}, Fail count = {}, Ignore count = {}",
244            self.pass_count, self.fail_count, self.ignore_count
245        );
246        println!("Testing time and memory = {}", self.stopwatch.elapsed());
247        report_metric("rustc failed tests", self.fail_count, "#");
248        report_metric("rustc testing time", self.stopwatch.elapsed().time.as_millis() as u64, "ms");
249    }
250}
251
252/// These tests break rust-analyzer (either by panicking or hanging) so we should ignore them.
253const IGNORED_TESTS: &[&str] = &[
254    "trait-with-missing-associated-type-restriction.rs", // #15646
255    "trait-with-missing-associated-type-restriction-fixable.rs", // #15646
256    "resolve-self-in-impl.rs",
257    "basic.rs", // ../rust/tests/ui/associated-type-bounds/return-type-notation/basic.rs
258    "issue-26056.rs",
259    "float-field.rs",
260    "invalid_operator_trait.rs",
261    "type-alias-impl-trait-assoc-dyn.rs",
262    "deeply-nested_closures.rs",    // exponential time
263    "hang-on-deeply-nested-dyn.rs", // exponential time
264    "dyn-rpit-and-let.rs", // unexpected free variable with depth `^1.0` with outer binder ^0
265    "issue-16098.rs",      // Huge recursion limit for macros?
266    "issue-83471.rs", // crates/hir-ty/src/builder.rs:78:9: assertion failed: self.remaining() > 0
267];
268
269const SUPPORTED_DIAGNOSTICS: &[DiagnosticCode] = &[
270    DiagnosticCode::RustcHardError("E0023"),
271    DiagnosticCode::RustcHardError("E0046"),
272    DiagnosticCode::RustcHardError("E0063"),
273    DiagnosticCode::RustcHardError("E0107"),
274    DiagnosticCode::RustcHardError("E0117"),
275    DiagnosticCode::RustcHardError("E0133"),
276    DiagnosticCode::RustcHardError("E0210"),
277    DiagnosticCode::RustcHardError("E0268"),
278    DiagnosticCode::RustcHardError("E0308"),
279    DiagnosticCode::RustcHardError("E0384"),
280    DiagnosticCode::RustcHardError("E0407"),
281    DiagnosticCode::RustcHardError("E0432"),
282    DiagnosticCode::RustcHardError("E0451"),
283    DiagnosticCode::RustcHardError("E0507"),
284    DiagnosticCode::RustcHardError("E0583"),
285    DiagnosticCode::RustcHardError("E0559"),
286    DiagnosticCode::RustcHardError("E0616"),
287    DiagnosticCode::RustcHardError("E0618"),
288    DiagnosticCode::RustcHardError("E0624"),
289    DiagnosticCode::RustcHardError("E0774"),
290    DiagnosticCode::RustcHardError("E0767"),
291    DiagnosticCode::RustcHardError("E0777"),
292];
293
294impl flags::RustcTests {
295    pub fn run(self) -> Result<()> {
296        let mut tester = Tester::new()?;
297        let walk_dir = WalkDir::new(self.rustc_repo.join("tests/ui"));
298        eprintln!("Running tests for tests/ui");
299        for i in walk_dir {
300            let i = i?;
301            let p = i.into_path();
302            if let Some(f) = &self.filter
303                && !p.as_os_str().to_string_lossy().contains(f)
304            {
305                continue;
306            }
307            if p.extension().is_none_or(|x| x != "rs") {
308                continue;
309            }
310            if let Err(e) = std::panic::catch_unwind({
311                let tester = AssertUnwindSafe(&mut tester);
312                let p = p.clone();
313                move || {
314                    let _guard = base_db::DbPanicContext::enter(p.display().to_string());
315                    { tester }.0.test(p);
316                }
317            }) {
318                std::panic::resume_unwind(e);
319            }
320        }
321        tester.report();
322        Ok(())
323    }
324}
ra_ap_rust_analyzer/cli/rustc_tests.rs

ra_ap_rust_analyzer/cli/
rustc_tests.rs
