Skip to main content

decl_lang/
conformance.rs

1//! Conformance judging — a port of the reference implementation's
2//! conformance.ts: judges fixtures by their declared phase.
3//!   valid/*                          -> parse + checks clean + outputs evaluate clean
4//!   invalid @expect-phase: parsing   -> must fail to parse
5//!   invalid @expect-phase: checking  -> parses; static checks report @expect-error
6//!   invalid @expect-phase: binding   -> parses; the pipeline reports @expect-error
7use crate::checker::check_module;
8use crate::parse::parse_source;
9use crate::pipeline::run_pipeline;
10use crate::semantics::Diag;
11use std::path::{Path, PathBuf};
12
13/// the judgment of one fixture
14pub struct Verdict {
15    /// the fixture's path
16    pub file: PathBuf,
17    /// whether it behaved as its header says
18    pub ok: bool,
19    /// why not: the diagnostics seen, or what was expected
20    pub detail: String,
21}
22
23/// Every `.decl` file under a directory, recursively and sorted, appended to `out`.
24pub fn walk_decl(dir: &Path, out: &mut Vec<PathBuf>) {
25    let Ok(rd) = std::fs::read_dir(dir) else {
26        return;
27    };
28    let mut entries: Vec<_> = rd.flatten().collect();
29    entries.sort_by_key(|e| e.file_name());
30    for e in entries {
31        let p = e.path();
32        if p.is_dir() {
33            walk_decl(&p, out);
34        } else if p.extension().map(|x| x == "decl").unwrap_or(false) {
35            out.push(p);
36        }
37    }
38}
39
40/// Judge one fixture (tests/validation/README.md): a valid one must parse, check,
41/// and evaluate clean; an invalid one must fail in the phase and with the code
42/// (and message) its `@expect-*` comments name.
43pub fn judge_fixture(file: &Path, is_valid: bool) -> Verdict {
44    let src = std::fs::read_to_string(file).unwrap_or_default();
45    let meta = |key: &str| -> Option<String> {
46        src.lines().find_map(|l| {
47            l.strip_prefix(&format!("// @{key}:"))
48                .map(|v| v.trim().to_string())
49        })
50    };
51    let phase = meta("expect-phase");
52    let want = meta("expect-error").unwrap_or_default();
53    let want_msg = meta("expect-message").unwrap_or_default();
54    let parsed = parse_source(&src);
55    let json_of = |ds: &[Diag]| {
56        format!(
57            "[{}]",
58            ds.iter()
59                .map(|d| d.to_json(None))
60                .collect::<Vec<_>>()
61                .join(",")
62        )
63    };
64    let hit = |ds: &[Diag]| {
65        ds.iter().any(|d| d.code.as_deref() == Some(want.as_str()))
66            && (want_msg.is_empty() || ds.iter().any(|d| d.message.contains(&want_msg)))
67    };
68    let verdict = |ok: bool, detail: String| Verdict {
69        file: file.to_path_buf(),
70        ok,
71        detail,
72    };
73    if is_valid {
74        // a valid fixture must parse, check clean, AND evaluate its outputs
75        // without error-severity diagnostics (a warning is not a failure)
76        if !parsed.errors.is_empty() {
77            return verdict(false, format!("{} parse errors", parsed.errors.len()));
78        }
79        let checks: Vec<Diag> = check_module(&parsed.decls, None, None)
80            .into_iter()
81            .filter(|d| d.severity == "error")
82            .collect();
83        let eval_errs: Vec<Diag> = if checks.is_empty() {
84            run_pipeline(&parsed.decls)
85                .diags
86                .into_iter()
87                .filter(|d| d.severity == "error")
88                .collect()
89        } else {
90            vec![]
91        };
92        let all: Vec<Diag> = checks.into_iter().chain(eval_errs).collect();
93        return verdict(all.is_empty(), json_of(&all));
94    }
95    match phase.as_deref() {
96        Some("parsing") => verdict(
97            !parsed.errors.is_empty(),
98            "expected parse errors, got none".into(),
99        ),
100        Some("checking") => {
101            let checks = if parsed.errors.is_empty() {
102                check_module(&parsed.decls, None, None)
103            } else {
104                vec![]
105            };
106            verdict(hit(&checks), json_of(&checks))
107        }
108        Some("binding") => {
109            let diags = if parsed.errors.is_empty() {
110                run_pipeline(&parsed.decls).diags
111            } else {
112                vec![]
113            };
114            verdict(hit(&diags), json_of(&diags))
115        }
116        other => verdict(false, format!("unknown phase {other:?}")),
117    }
118}
119
120/// Judge every fixture under a directory: `valid/` paths as valid, the rest as invalid.
121pub fn judge_corpus(dir: &Path) -> Vec<Verdict> {
122    let mut files = vec![];
123    walk_decl(dir, &mut files);
124    files
125        .iter()
126        .map(|f| judge_fixture(f, f.to_string_lossy().contains("/valid/")))
127        .collect()
128}