sim_lib_standard_core/
matrix.rs

1//! Shared language conformance matrix data structures.
2
3use indexmap::IndexMap;
4use sim_kernel::{Cx, Error, Expr, Result, Symbol, Value};
5
6use crate::{
7    ConformanceOutcome, LanguageProfile, matrix_claims::publish_matrix_cell_claim,
8    standard_test_capability,
9};
10
11/// Expected outcome for a source-level conformance case.
12#[derive(Clone, Debug, PartialEq, Eq)]
13pub enum SourceExpectation {
14    /// The source lowers to the described shared expression form.
15    LowersTo(String),
16    /// The source is an explicit known gap with a machine-readable code.
17    ExpectedGap {
18        /// Gap code.
19        code: Symbol,
20        /// Human-readable reason.
21        reason: String,
22    },
23}
24
25/// Observation produced by a language-specific source-case runner.
26#[derive(Clone, Debug, PartialEq, Eq)]
27pub enum SourceObservation {
28    /// Source lowered to the displayed shared form.
29    LowersTo(String),
30    /// Source is a declared gap with a machine-readable code and reason.
31    Gap {
32        /// Gap code.
33        code: Symbol,
34        /// Human-readable reason.
35        reason: String,
36    },
37}
38
39/// Whether a source conformance case is scored or descriptor-only.
40#[derive(Clone, Copy, Debug, PartialEq, Eq)]
41pub enum SourceConformanceCaseKind {
42    /// A scored case that must be backed by observed runtime behavior.
43    Observed,
44    /// A descriptor-only case that remains visible but does not affect scoring.
45    DescriptorOnly,
46}
47
48impl SourceConformanceCaseKind {
49    fn cell_kind(self) -> MatrixCellKind {
50        match self {
51            Self::Observed => MatrixCellKind::SourceObserved,
52            Self::DescriptorOnly => MatrixCellKind::DescriptorOnly,
53        }
54    }
55}
56
57/// One source-language conformance case.
58#[derive(Clone, Debug, PartialEq, Eq)]
59pub struct SourceConformanceCase {
60    /// Stable symbol identifying this case.
61    pub symbol: Symbol,
62    /// Organ exercised by this case.
63    pub organ: Symbol,
64    /// Source filename or display name.
65    pub source_name: String,
66    /// Source text.
67    pub source: String,
68    /// Whether this case is scored or descriptor-only.
69    pub kind: SourceConformanceCaseKind,
70    /// Expected result.
71    pub expectation: SourceExpectation,
72    /// Fidelity badge affected by this case, if any.
73    pub affects_badge: Option<Symbol>,
74}
75
76/// Codec-faithful source case that decodes to the shared `Expr` graph.
77///
78/// The case records source text plus the canonical display expected from the
79/// decoded expression. A missing expected display marks an explicit gap case;
80/// a language-specific decoder returns `Ok(None)` when that gap is observed.
81#[derive(Clone, Debug, PartialEq, Eq)]
82pub struct ExprRoundTripCase {
83    /// Stable symbol identifying this case.
84    pub symbol: Symbol,
85    /// Language exercised by this case.
86    pub language: Symbol,
87    /// Source text.
88    pub source: String,
89    /// Expected canonical display of the decoded expression.
90    pub expected_display: Option<String>,
91    /// Fidelity badge affected by this case, if any.
92    pub affects_badge: Option<Symbol>,
93}
94
95/// Observation produced by running an [`ExprRoundTripCase`].
96#[derive(Clone, Debug, PartialEq, Eq)]
97pub enum ExprRoundTripObservation {
98    /// Decoded and matched the expected display, or no display was required.
99    RoundTripped(String),
100    /// Decoded but did not match the expected display.
101    Mismatch {
102        /// Expected expression display.
103        expected: String,
104        /// Actual expression display.
105        got: String,
106    },
107    /// Codec returned a diagnostic code.
108    Diagnostic(Symbol),
109    /// Known gap; decode was not attempted.
110    Gap(Symbol),
111}
112
113impl ExprRoundTripCase {
114    /// Runs this case using `decode_fn` to decode source into an expression.
115    pub fn run_expr_round_trip(
116        &self,
117        cx: &mut Cx,
118        decode_fn: impl Fn(&mut Cx, &str) -> Result<Option<Expr>>,
119    ) -> ExprRoundTripObservation {
120        match decode_fn(cx, &self.source) {
121            Err(err) => ExprRoundTripObservation::Diagnostic(Symbol::qualified(
122                "codec",
123                diagnostic_slug(&err),
124            )),
125            Ok(None) => ExprRoundTripObservation::Gap(Symbol::qualified("codec", "declared-gap")),
126            Ok(Some(expr)) => {
127                let got = expr_display(&expr);
128                match &self.expected_display {
129                    None => ExprRoundTripObservation::RoundTripped(got),
130                    Some(expected) if expected == &got => {
131                        ExprRoundTripObservation::RoundTripped(got)
132                    }
133                    Some(expected) => ExprRoundTripObservation::Mismatch {
134                        expected: expected.clone(),
135                        got,
136                    },
137                }
138            }
139        }
140    }
141
142    /// Runs this case using `decode_fn` to decode source into an expression.
143    pub fn run(
144        &self,
145        cx: &mut Cx,
146        decode_fn: impl Fn(&mut Cx, &str) -> Result<Option<Expr>>,
147    ) -> ExprRoundTripObservation {
148        self.run_expr_round_trip(cx, decode_fn)
149    }
150}
151
152/// A single language surface registered in the shared conformance matrix.
153///
154/// The row contains current conformance evidence for one language profile. Each
155/// row uses a stable language symbol, owns the profile metadata for that row,
156/// and carries only explicit source or expression cases. An empty row is a
157/// declared language entry without scored evidence.
158#[derive(Clone, Debug, PartialEq, Eq)]
159pub struct LanguageRow {
160    /// Language symbol, for example `scheme` or `lua`.
161    pub language: Symbol,
162    /// Profile supplied by the language crate.
163    pub profile: LanguageProfile,
164    /// Source cases registered for this language.
165    pub cases: Vec<SourceConformanceCase>,
166    /// Expression round-trip cases registered for this language.
167    pub expr_cases: Vec<ExprRoundTripCase>,
168}
169
170impl LanguageRow {
171    /// Declares a language row with no source cases.
172    pub fn declared_empty(language: Symbol, profile: LanguageProfile) -> Self {
173        Self {
174            language,
175            profile,
176            cases: Vec::new(),
177            expr_cases: Vec::new(),
178        }
179    }
180
181    /// Returns whether this row currently has no cases.
182    pub fn is_empty(&self) -> bool {
183        self.cases.is_empty() && self.expr_cases.is_empty()
184    }
185
186    /// Replaces expression round-trip cases for this row.
187    pub fn with_expr_cases(mut self, expr_cases: Vec<ExprRoundTripCase>) -> Self {
188        self.expr_cases = expr_cases;
189        self
190    }
191}
192
193/// Builder for [`LanguageRow`] values.
194#[derive(Clone, Debug)]
195pub struct LanguageRowBuilder {
196    language: Symbol,
197    profile: LanguageProfile,
198    cases: Vec<SourceConformanceCase>,
199    expr_cases: Vec<ExprRoundTripCase>,
200}
201
202impl LanguageRowBuilder {
203    /// Starts a row builder for `language` and `profile`.
204    pub fn new(language: Symbol, profile: LanguageProfile) -> Self {
205        Self {
206            language,
207            profile,
208            cases: Vec::new(),
209            expr_cases: Vec::new(),
210        }
211    }
212
213    /// Appends one source case.
214    pub fn with_case(mut self, case: SourceConformanceCase) -> Self {
215        self.cases.push(case);
216        self
217    }
218
219    /// Appends source cases from an iterator.
220    pub fn with_cases<I>(mut self, cases: I) -> Self
221    where
222        I: IntoIterator<Item = SourceConformanceCase>,
223    {
224        self.cases.extend(cases);
225        self
226    }
227
228    /// Appends expression round-trip cases from an iterator.
229    pub fn with_expr_cases<I>(mut self, cases: I) -> Self
230    where
231        I: IntoIterator<Item = ExprRoundTripCase>,
232    {
233        self.expr_cases.extend(cases);
234        self
235    }
236
237    /// Builds the row.
238    pub fn build(self) -> LanguageRow {
239        LanguageRow {
240            language: self.language,
241            profile: self.profile,
242            cases: self.cases,
243            expr_cases: self.expr_cases,
244        }
245    }
246}
247
248/// Published kind for one matrix result cell.
249#[derive(Clone, Copy, Debug, PartialEq, Eq)]
250pub enum MatrixCellKind {
251    /// A scored source case backed by observed runtime behavior.
252    SourceObserved,
253    /// A descriptor-only source case that remains visible but unscored.
254    DescriptorOnly,
255    /// A scored expression round-trip case.
256    ExprRoundTrip,
257    /// A generated coverage report published through the matrix claim surface.
258    GeneratedCoverage,
259}
260
261impl MatrixCellKind {
262    /// Symbol stored in published evidence for this cell kind.
263    pub fn symbol(self) -> Symbol {
264        match self {
265            Self::SourceObserved => Symbol::qualified("standard-test", "source-observed"),
266            Self::DescriptorOnly => Symbol::qualified("standard-test", "descriptor-only"),
267            Self::ExprRoundTrip => Symbol::qualified("standard-test", "expr-round-trip"),
268            Self::GeneratedCoverage => Symbol::qualified("standard-test", "generated-coverage"),
269        }
270    }
271
272    fn is_scored(self) -> bool {
273        matches!(self, Self::SourceObserved | Self::ExprRoundTrip)
274    }
275}
276
277/// Outcome for a single language/case cell in a matrix run.
278#[derive(Clone, Debug, PartialEq, Eq)]
279pub struct MatrixCellResult {
280    /// Language symbol for this row.
281    pub language: Symbol,
282    /// Profile symbol for this row.
283    pub profile: Symbol,
284    /// Organ exercised by this case.
285    pub organ: Symbol,
286    /// Stable case symbol.
287    pub case_symbol: Symbol,
288    /// Kind of evidence this cell contributes.
289    pub kind: MatrixCellKind,
290    /// Fidelity badge affected by this cell, if any.
291    pub affects_badge: Option<Symbol>,
292    /// Compared conformance outcome.
293    pub outcome: ConformanceOutcome,
294}
295
296impl MatrixCellResult {
297    fn is_scored(&self) -> bool {
298        self.kind.is_scored()
299    }
300}
301
302/// Accumulated results for one matrix run.
303///
304/// The report is evidence produced by a runner invocation. Gaps remain visible
305/// as cells, while fidelity counts only pass and fail cells so declared gaps do
306/// not inflate or reduce the score.
307#[derive(Clone, Debug, PartialEq, Eq)]
308pub struct MatrixRunReport {
309    /// Matrix cells produced by the run.
310    pub cells: Vec<MatrixCellResult>,
311}
312
313impl MatrixRunReport {
314    /// Number of passing cells.
315    pub fn pass_count(&self) -> usize {
316        self.cells
317            .iter()
318            .filter(|cell| cell.is_scored() && cell.outcome.is_pass())
319            .count()
320    }
321
322    /// Number of declared gap cells.
323    pub fn gap_count(&self) -> usize {
324        self.cells
325            .iter()
326            .filter(|cell| cell.is_scored() && cell.outcome.is_gap())
327            .count()
328    }
329
330    /// Number of failing cells.
331    pub fn fail_count(&self) -> usize {
332        self.cells
333            .iter()
334            .filter(|cell| cell.is_scored() && cell.outcome.is_fail())
335            .count()
336    }
337
338    /// Fidelity for one language: passes divided by passes plus failures,
339    /// ignoring declared gaps. Returns `None` when no pass-or-fail cells exist.
340    pub fn language_fidelity(&self, language: &Symbol) -> Option<f32> {
341        let pass = self
342            .cells
343            .iter()
344            .filter(|cell| &cell.language == language && cell.is_scored() && cell.outcome.is_pass())
345            .count();
346        let fail = self
347            .cells
348            .iter()
349            .filter(|cell| &cell.language == language && cell.is_scored() && cell.outcome.is_fail())
350            .count();
351        if pass + fail == 0 {
352            None
353        } else {
354            Some(pass as f32 / (pass + fail) as f32)
355        }
356    }
357
358    /// Produces Card fields for one language's browseable conformance surface.
359    ///
360    /// These fields answer how much of a language profile is backed by current
361    /// matrix evidence for agents and humans browsing the Card.
362    pub fn conformance_card_fields(
363        &self,
364        cx: &mut Cx,
365        language: &Symbol,
366    ) -> Result<Vec<(Symbol, Value)>> {
367        let pass = self.language_outcome_count(language, ConformanceOutcome::is_pass);
368        let gap = self.language_outcome_count(language, ConformanceOutcome::is_gap);
369        let fail = self.language_outcome_count(language, ConformanceOutcome::is_fail);
370        let fidelity = self
371            .language_fidelity(language)
372            .map(|value| format!("{:.0}%", value * 100.0))
373            .unwrap_or_else(|| "unscored".to_owned());
374        conformance_card_fields(cx, pass, gap, fail, fidelity)
375    }
376
377    /// Produces zero-count conformance Card fields with unscored fidelity.
378    pub fn unscored_conformance_card_fields(cx: &mut Cx) -> Result<Vec<(Symbol, Value)>> {
379        conformance_card_fields(cx, 0, 0, 0, "unscored".to_owned())
380    }
381
382    /// Writes one evidence claim per cell into the claim store.
383    pub fn publish_claims(&self, cx: &mut Cx) -> Result<()> {
384        cx.require(&standard_test_capability())?;
385        for cell in &self.cells {
386            publish_matrix_cell_claim(cx, cell)?;
387        }
388        Ok(())
389    }
390
391    fn language_outcome_count(
392        &self,
393        language: &Symbol,
394        matches: impl Fn(&ConformanceOutcome) -> bool,
395    ) -> usize {
396        self.cells
397            .iter()
398            .filter(|cell| &cell.language == language && cell.is_scored() && matches(&cell.outcome))
399            .count()
400    }
401}
402
403/// Runs language rows through caller-supplied source-case runners.
404///
405/// The runner compares the row's expected source outcomes with observations
406/// from the caller. It does not depend on a concrete language codec; each
407/// language crate supplies its own execution closure and publishes the report
408/// when evidence claims are needed.
409pub struct MatrixRunner;
410
411impl MatrixRunner {
412    /// Runs a single language row that has source cases only.
413    pub fn run_source_row<F>(cx: &mut Cx, row: &LanguageRow, run_case: F) -> MatrixRunReport
414    where
415        F: Fn(&mut Cx, &SourceConformanceCase) -> Result<SourceObservation>,
416    {
417        Self::run_row(cx, row, run_case, |_cx, _case| {
418            panic!("source-only row attempted to execute expression cases")
419        })
420    }
421
422    /// Runs a single language row, using caller-supplied closures for both
423    /// source cases and expression round-trip cases.
424    pub fn run_row<F, G>(
425        cx: &mut Cx,
426        row: &LanguageRow,
427        run_source_case: F,
428        run_expr_case: G,
429    ) -> MatrixRunReport
430    where
431        F: Fn(&mut Cx, &SourceConformanceCase) -> Result<SourceObservation>,
432        G: Fn(&mut Cx, &ExprRoundTripCase) -> Result<ExprRoundTripObservation>,
433    {
434        let mut cells = Vec::with_capacity(row.cases.len() + row.expr_cases.len());
435        for case in &row.cases {
436            let outcome = match run_source_case(cx, case) {
437                Ok(observation) => compare_source_observation(case, observation),
438                Err(err) => ConformanceOutcome::fail_with(err.to_string()),
439            };
440            cells.push(MatrixCellResult {
441                language: row.language.clone(),
442                profile: row.profile.symbol.clone(),
443                organ: case.organ.clone(),
444                case_symbol: case.symbol.clone(),
445                kind: case.kind.cell_kind(),
446                affects_badge: case.affects_badge.clone(),
447                outcome,
448            });
449        }
450        for case in &row.expr_cases {
451            let outcome = match run_expr_case(cx, case) {
452                Ok(observation) => compare_expr_observation(case, observation),
453                Err(err) => ConformanceOutcome::fail_with(err.to_string()),
454            };
455            cells.push(MatrixCellResult {
456                language: row.language.clone(),
457                profile: row.profile.symbol.clone(),
458                organ: expr_round_trip_organ(&row.language),
459                case_symbol: case.symbol.clone(),
460                kind: MatrixCellKind::ExprRoundTrip,
461                affects_badge: case.affects_badge.clone(),
462                outcome,
463            });
464        }
465        MatrixRunReport { cells }
466    }
467}
468
469/// Compares a source observation against its expected result.
470pub fn compare_source_observation(
471    case: &SourceConformanceCase,
472    observation: SourceObservation,
473) -> ConformanceOutcome {
474    match (&case.expectation, observation) {
475        (SourceExpectation::LowersTo(expected), SourceObservation::LowersTo(got)) => {
476            if expected == &got {
477                ConformanceOutcome::pass()
478            } else {
479                ConformanceOutcome::fail(format!("expected {expected}, got {got}"))
480            }
481        }
482        (
483            SourceExpectation::ExpectedGap { code, reason },
484            SourceObservation::Gap {
485                code: got,
486                reason: got_reason,
487            },
488        ) => {
489            if code == &got {
490                ConformanceOutcome::gap(reason.clone())
491            } else {
492                ConformanceOutcome::fail(format!(
493                    "expected gap {code}, got gap {got}: {got_reason}"
494                ))
495            }
496        }
497        (SourceExpectation::ExpectedGap { code, .. }, SourceObservation::LowersTo(got)) => {
498            ConformanceOutcome::fail(format!("expected gap {code}, got {got}"))
499        }
500        (SourceExpectation::LowersTo(expected), SourceObservation::Gap { code, reason }) => {
501            ConformanceOutcome::fail(format!("expected {expected}, got gap {code}: {reason}"))
502        }
503    }
504}
505
506/// Compares an expression round-trip observation against its expected result.
507pub fn compare_expr_observation(
508    case: &ExprRoundTripCase,
509    observation: ExprRoundTripObservation,
510) -> ConformanceOutcome {
511    match (&case.expected_display, observation) {
512        (Some(_), ExprRoundTripObservation::RoundTripped(_)) => ConformanceOutcome::pass(),
513        (Some(_), ExprRoundTripObservation::Mismatch { expected, got }) => {
514            ConformanceOutcome::fail(format!("expected {expected}, got {got}"))
515        }
516        (Some(expected), ExprRoundTripObservation::Diagnostic(code)) => {
517            ConformanceOutcome::fail(format!("expected {expected}, got diagnostic {code}"))
518        }
519        (Some(expected), ExprRoundTripObservation::Gap(code)) => {
520            ConformanceOutcome::fail(format!("expected {expected}, got gap {code}"))
521        }
522        (None, ExprRoundTripObservation::Gap(code)) => ConformanceOutcome::gap(code.to_string()),
523        (None, ExprRoundTripObservation::RoundTripped(got)) => {
524            ConformanceOutcome::fail(format!("expected declared gap, got {got}"))
525        }
526        (None, ExprRoundTripObservation::Diagnostic(code)) => {
527            ConformanceOutcome::fail(format!("expected declared gap, got diagnostic {code}"))
528        }
529        (None, ExprRoundTripObservation::Mismatch { expected, got }) => {
530            ConformanceOutcome::fail(format!("expected declared gap, got {expected} -> {got}"))
531        }
532    }
533}
534
535/// Shared conformance matrix keyed by language symbol.
536///
537/// Rows preserve registration order and are unique by language symbol. The
538/// matrix owns row metadata and case definitions only; execution lives in
539/// [`MatrixRunner`] and language-specific runners.
540#[derive(Default)]
541pub struct ConformanceMatrix {
542    rows: IndexMap<Symbol, LanguageRow>,
543}
544
545impl ConformanceMatrix {
546    /// Creates an empty matrix.
547    pub fn new() -> Self {
548        Self::default()
549    }
550
551    /// Registers a language row.
552    ///
553    /// # Panics
554    ///
555    /// Panics when the language symbol is already registered.
556    pub fn register(&mut self, row: LanguageRow) {
557        let language = row.language.clone();
558        assert!(
559            self.rows.insert(language.clone(), row).is_none(),
560            "language already registered in matrix: {language}",
561        );
562    }
563
564    /// Number of registered languages.
565    pub fn language_count(&self) -> usize {
566        self.rows.len()
567    }
568
569    /// Returns the row for `language`, if registered.
570    pub fn row(&self, language: &Symbol) -> Option<&LanguageRow> {
571        self.rows.get(language)
572    }
573
574    /// Iterates rows in registration order.
575    pub fn iter_rows(&self) -> impl Iterator<Item = &LanguageRow> {
576        self.rows.values()
577    }
578
579    /// Total source cases across all registered languages.
580    pub fn total_cases(&self) -> usize {
581        self.rows.values().map(|row| row.cases.len()).sum()
582    }
583
584    /// Total expression round-trip cases across all registered languages.
585    pub fn total_expr_cases(&self) -> usize {
586        self.rows.values().map(|row| row.expr_cases.len()).sum()
587    }
588}
589
590fn expr_display(expr: &Expr) -> String {
591    format!("Expr::{expr:?}")
592}
593
594fn expr_round_trip_organ(language: &Symbol) -> Symbol {
595    Symbol::qualified(language.as_qualified_str(), "expr-round-trip")
596}
597
598fn diagnostic_slug(err: &Error) -> &'static str {
599    if err.to_string().to_ascii_lowercase().contains("unsupported") {
600        "unsupported"
601    } else {
602        "error"
603    }
604}
605
606fn conformance_card_fields(
607    cx: &mut Cx,
608    pass: usize,
609    gap: usize,
610    fail: usize,
611    fidelity: String,
612) -> Result<Vec<(Symbol, Value)>> {
613    Ok(vec![
614        (conformance_field("pass"), count_value(cx, pass)?),
615        (conformance_field("gap"), count_value(cx, gap)?),
616        (conformance_field("fail"), count_value(cx, fail)?),
617        (
618            conformance_field("fidelity"),
619            cx.factory().string(fidelity)?,
620        ),
621    ])
622}
623
624fn conformance_field(name: &str) -> Symbol {
625    Symbol::new(format!("conformance.{name}"))
626}
627
628fn count_value(cx: &mut Cx, count: usize) -> Result<Value> {
629    cx.factory()
630        .number_literal(Symbol::qualified("numbers", "u64"), count.to_string())
631}
sim_lib_standard_core/matrix.rs

sim_lib_standard_core/
matrix.rs
