sim_lib_lang_genconf/
coverage.rs

1//! Measured coverage reports for generated expression conformance.
2
3use sim_codec::{Input, decode_with_codec};
4use sim_kernel::{Cx, Expr, ReadPolicy, Symbol};
5use sim_lib_standard_core::{
6    ExprRoundTripCase, ExprRoundTripObservation, LanguageProfile, LanguageRowBuilder, MatrixRunner,
7    SourceObservation,
8};
9
10use crate::property::{check_round_trip, generated_expr_cases};
11use crate::space::ExprSpace;
12
13/// A reproducible measurement of generated expression round-trip coverage.
14#[derive(Clone, Debug, PartialEq, Eq)]
15pub struct GeneratedCoverageReport {
16    /// Language row measured by this report.
17    pub language: Symbol,
18    /// Number of generated cases sampled.
19    pub sampled: usize,
20    /// Number of sampled cases that round-tripped.
21    pub round_tripped: usize,
22    /// Number of sampled cases that decoded to a different expression.
23    pub mismatched: usize,
24    /// Number of sampled cases that produced diagnostics or declared gaps.
25    pub diagnostics: usize,
26    /// Maximum expression depth used by the space.
27    pub max_depth: usize,
28    /// Seed corpus used as the landmark gate for this measurement.
29    pub seed: Vec<Expr>,
30    /// Whether every seed landmark round-tripped before coverage reporting.
31    pub landmark_reproduced: bool,
32    /// Seed expressions that did not round-trip before measurement.
33    pub unmet_landmarks: Vec<Expr>,
34}
35
36impl GeneratedCoverageReport {
37    /// Returns true when every seed landmark round-tripped.
38    pub fn landmark_reproduced(&self) -> bool {
39        self.landmark_reproduced
40    }
41
42    /// Coverage ratio, `round_tripped / sampled`.
43    ///
44    /// Returns `None` when seed landmark reproduction did not pass or the run
45    /// sampled no generated cases.
46    pub fn coverage(&self) -> Option<f32> {
47        if !self.landmark_reproduced() || self.sampled == 0 {
48            return None;
49        }
50        Some(self.round_tripped as f32 / self.sampled as f32)
51    }
52
53    /// Coverage percentage, derived from [`GeneratedCoverageReport::coverage`].
54    pub fn coverage_percent(&self) -> Option<f32> {
55        self.coverage().map(|ratio| ratio * 100.0)
56    }
57}
58
59/// Runs generated expression cases for one language and codec.
60///
61/// The generated cases are attached to a standard language row and the row is
62/// passed through [`MatrixRunner`]. Expression observations are counted through
63/// the existing [`ExprRoundTripCase`] path.
64pub fn run_generated_row(
65    cx: &mut Cx,
66    language: &Symbol,
67    codec: &Symbol,
68    space: &ExprSpace,
69    budget: usize,
70) -> GeneratedCoverageReport {
71    let seed = space.seed_corpus();
72    let unmet_landmarks = unmet_landmarks(cx, codec, &seed);
73    let generated_cases = generated_expr_cases(cx, language, codec, space, budget);
74    let row = LanguageRowBuilder::new(
75        language.clone(),
76        LanguageProfile::new(Symbol::qualified(
77            "lang/generated",
78            language.as_qualified_str().to_owned(),
79        )),
80    )
81    .with_expr_cases(generated_cases)
82    .build();
83    let matrix_report = MatrixRunner::run_row(cx, &row, |_cx, _case| {
84        Ok(SourceObservation::LowersTo(String::new()))
85    });
86    debug_assert!(matrix_report.cells.is_empty());
87
88    let mut round_tripped = 0;
89    let mut mismatched = 0;
90    let mut diagnostics = 0;
91    for case in &row.expr_cases {
92        match run_expr_case(cx, codec, case) {
93            ExprRoundTripObservation::RoundTripped(_) => round_tripped += 1,
94            ExprRoundTripObservation::Mismatch { .. } => mismatched += 1,
95            ExprRoundTripObservation::Diagnostic(_) | ExprRoundTripObservation::Gap(_) => {
96                diagnostics += 1;
97            }
98        }
99    }
100
101    GeneratedCoverageReport {
102        language: language.clone(),
103        sampled: row.expr_cases.len(),
104        round_tripped,
105        mismatched,
106        diagnostics,
107        max_depth: space.max_depth(),
108        seed,
109        landmark_reproduced: unmet_landmarks.is_empty(),
110        unmet_landmarks,
111    }
112}
113
114fn unmet_landmarks(cx: &mut Cx, codec: &Symbol, seed: &[Expr]) -> Vec<Expr> {
115    seed.iter()
116        .filter(|expr| {
117            !matches!(
118                check_round_trip(cx, codec, expr),
119                ExprRoundTripObservation::RoundTripped(_)
120            )
121        })
122        .cloned()
123        .collect()
124}
125
126fn run_expr_case(
127    cx: &mut Cx,
128    codec: &Symbol,
129    case: &ExprRoundTripCase,
130) -> ExprRoundTripObservation {
131    case.run(cx, |cx, source| {
132        decode_with_codec(
133            cx,
134            codec,
135            Input::Text(source.to_owned()),
136            ReadPolicy::default(),
137        )
138        .map(Some)
139    })
140}
141
142#[cfg(test)]
143mod tests {
144    use std::sync::Arc;
145
146    use sim_kernel::{DefaultFactory, EagerPolicy};
147    use sim_lib_lang_scheme::{SchemeCodecLib, scheme_reader_symbol};
148
149    use super::*;
150
151    fn coverage_cx() -> Cx {
152        let mut cx = Cx::new(Arc::new(EagerPolicy), Arc::new(DefaultFactory));
153        sim_test_support::register_core_classes(&mut cx);
154        sim_test_support::register_f64_number_domain(&mut cx);
155        cx
156    }
157
158    fn register_scheme_codec(cx: &mut Cx) -> Symbol {
159        let codec = scheme_reader_symbol();
160        let lib = SchemeCodecLib::new(cx.registry_mut().fresh_codec_id());
161        cx.load_lib(&lib).unwrap();
162        codec
163    }
164
165    #[test]
166    fn scheme_generated_coverage_is_reproducible() {
167        let mut first_cx = coverage_cx();
168        let mut second_cx = coverage_cx();
169        let first_codec = register_scheme_codec(&mut first_cx);
170        let second_codec = register_scheme_codec(&mut second_cx);
171        let language = Symbol::new("scheme");
172        let space = ExprSpace::r7rs_core_space(3);
173
174        let first = run_generated_row(&mut first_cx, &language, &first_codec, &space, 8);
175        let second = run_generated_row(&mut second_cx, &language, &second_codec, &space, 8);
176
177        assert_eq!(first, second);
178        assert_eq!(first.language, language);
179        assert_eq!(first.sampled, 8);
180        assert_eq!(first.round_tripped, 0);
181        assert_eq!(first.mismatched, 0);
182        assert_eq!(first.diagnostics, 8);
183        assert_eq!(first.max_depth, 3);
184        assert_eq!(first.seed.len(), 5);
185        assert_eq!(first.unmet_landmarks.len(), first.seed.len());
186        assert_eq!(first.coverage(), None);
187    }
188
189    #[test]
190    fn coverage_is_none_without_landmark_reproduction() {
191        let mut cx = coverage_cx();
192        let codec = register_scheme_codec(&mut cx);
193        let report = run_generated_row(
194            &mut cx,
195            &Symbol::new("scheme"),
196            &codec,
197            &ExprSpace::r7rs_core_space(2),
198            4,
199        );
200
201        assert!(!report.landmark_reproduced());
202        assert_eq!(report.coverage(), None);
203        assert!(!report.unmet_landmarks.is_empty());
204    }
205
206    #[test]
207    fn coverage_ratio_is_round_tripped_over_sampled_after_landmarks() {
208        let report = GeneratedCoverageReport {
209            language: Symbol::new("scheme"),
210            sampled: 4,
211            round_tripped: 3,
212            mismatched: 1,
213            diagnostics: 0,
214            max_depth: 2,
215            seed: vec![Expr::Bool(true)],
216            landmark_reproduced: true,
217            unmet_landmarks: Vec::new(),
218        };
219
220        assert!(report.landmark_reproduced());
221        assert_eq!(report.coverage(), Some(0.75));
222        assert_eq!(report.coverage_percent(), Some(75.0));
223    }
224}
sim_lib_lang_genconf/coverage.rs

sim_lib_lang_genconf/
coverage.rs
