decl_lang/
conformance.rs

1//! Conformance judging — a port of the reference implementation's
2//! conformance.ts: judges fixtures by their declared phase.
3//!   valid/*                          -> parse + checks clean + outputs evaluate clean
4//!   invalid @expect-phase: parsing   -> must fail to parse
5//!   invalid @expect-phase: checking  -> parses; static checks report @expect-error
6//!   invalid @expect-phase: binding   -> parses; the pipeline reports @expect-error
7use crate::checker::check_module;
8use crate::parse::parse_source;
9use crate::pipeline::run_pipeline;
10use crate::semantics::Diag;
11use std::path::{Path, PathBuf};
12
13pub struct Verdict {
14    pub file: PathBuf,
15    pub ok: bool,
16    pub detail: String,
17}
18
19pub fn walk_decl(dir: &Path, out: &mut Vec<PathBuf>) {
20    let Ok(rd) = std::fs::read_dir(dir) else {
21        return;
22    };
23    let mut entries: Vec<_> = rd.flatten().collect();
24    entries.sort_by_key(|e| e.file_name());
25    for e in entries {
26        let p = e.path();
27        if p.is_dir() {
28            walk_decl(&p, out);
29        } else if p.extension().map(|x| x == "decl").unwrap_or(false) {
30            out.push(p);
31        }
32    }
33}
34
35pub fn judge_fixture(file: &Path, is_valid: bool) -> Verdict {
36    let src = std::fs::read_to_string(file).unwrap_or_default();
37    let meta = |key: &str| -> Option<String> {
38        src.lines().find_map(|l| {
39            l.strip_prefix(&format!("// @{key}:"))
40                .map(|v| v.trim().to_string())
41        })
42    };
43    let phase = meta("expect-phase");
44    let want = meta("expect-error").unwrap_or_default();
45    let want_msg = meta("expect-message").unwrap_or_default();
46    let parsed = parse_source(&src);
47    let json_of = |ds: &[Diag]| {
48        format!(
49            "[{}]",
50            ds.iter()
51                .map(|d| d.to_json(None))
52                .collect::<Vec<_>>()
53                .join(",")
54        )
55    };
56    let hit = |ds: &[Diag]| {
57        ds.iter().any(|d| d.code.as_deref() == Some(want.as_str()))
58            && (want_msg.is_empty() || ds.iter().any(|d| d.message.contains(&want_msg)))
59    };
60    let verdict = |ok: bool, detail: String| Verdict {
61        file: file.to_path_buf(),
62        ok,
63        detail,
64    };
65    if is_valid {
66        // a valid fixture must parse, check clean, AND evaluate its outputs
67        // without error-severity diagnostics
68        if !parsed.errors.is_empty() {
69            return verdict(false, format!("{} parse errors", parsed.errors.len()));
70        }
71        let checks = check_module(&parsed.decls, None, None);
72        let eval_errs: Vec<Diag> = if checks.is_empty() {
73            run_pipeline(&parsed.decls)
74                .diags
75                .into_iter()
76                .filter(|d| d.severity == "error")
77                .collect()
78        } else {
79            vec![]
80        };
81        let all: Vec<Diag> = checks.into_iter().chain(eval_errs).collect();
82        return verdict(all.is_empty(), json_of(&all));
83    }
84    match phase.as_deref() {
85        Some("parsing") => verdict(
86            !parsed.errors.is_empty(),
87            "expected parse errors, got none".into(),
88        ),
89        Some("checking") => {
90            let checks = if parsed.errors.is_empty() {
91                check_module(&parsed.decls, None, None)
92            } else {
93                vec![]
94            };
95            verdict(hit(&checks), json_of(&checks))
96        }
97        Some("binding") => {
98            let diags = if parsed.errors.is_empty() {
99                run_pipeline(&parsed.decls).diags
100            } else {
101                vec![]
102            };
103            verdict(hit(&diags), json_of(&diags))
104        }
105        other => verdict(false, format!("unknown phase {other:?}")),
106    }
107}
108
109pub fn judge_corpus(dir: &Path) -> Vec<Verdict> {
110    let mut files = vec![];
111    walk_decl(dir, &mut files);
112    files
113        .iter()
114        .map(|f| judge_fixture(f, f.to_string_lossy().contains("/valid/")))
115        .collect()
116}
decl_lang/conformance.rs

decl_lang/
conformance.rs
