mcp_trace_validator/
report.rs

1// SPDX-License-Identifier: MIT
2// Copyright 2026 Tom F. (https://github.com/tomtom215)
3
4//! Validation reports: per-requirement outcomes with actionable findings.
5//!
6//! Reports are artifacts: they get committed as golden files, diffed in CI, and cited
7//! in published results. Two consequences shape this module: serialization order is
8//! fixed (registry order; struct fields in declaration order), and nothing
9//! environment-dependent (paths, timestamps, hostnames) is ever included.
10
11use core::fmt;
12use core::fmt::Write as _;
13
14use serde::{Deserialize, Serialize};
15
16/// One concrete violation, addressed to a requirement and (where possible) an event.
17#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
18#[non_exhaustive]
19pub struct Finding {
20    /// The validator check that produced this finding.
21    pub check: String,
22    /// The event `seq` the finding points at, when one event is identifiable.
23    #[serde(default, skip_serializing_if = "Option::is_none")]
24    pub seq: Option<u64>,
25    /// What was observed and what was expected, in one actionable sentence.
26    pub detail: String,
27}
28
29/// The outcome of evaluating one requirement against one trace.
30#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
31#[serde(rename_all = "kebab-case")]
32#[non_exhaustive]
33pub enum Outcome {
34    /// All covering checks ran and produced no findings.
35    Pass,
36    /// A MUST / MUST NOT requirement has findings.
37    Fail,
38    /// A SHOULD / SHOULD NOT requirement has findings.
39    Warn,
40    /// The registry documents that this requirement is not judged from traces.
41    Excluded,
42    /// The registry references a check this validator build does not implement.
43    Unsupported,
44    /// The requirement is gated on a capability this session never declared
45    /// (ADR-0006); its checks were not run.
46    NotApplicable,
47}
48
49/// Aggregate counts, in report order. `excluded` and `unsupported` are first-class:
50/// inflating pass rates by hiding them is how conformance tools lose trust.
51#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
52#[non_exhaustive]
53pub struct Totals {
54    /// Requirements with outcome [`Outcome::Pass`].
55    pub pass: u32,
56    /// Requirements with outcome [`Outcome::Fail`].
57    pub fail: u32,
58    /// Requirements with outcome [`Outcome::Warn`].
59    pub warn: u32,
60    /// Requirements with outcome [`Outcome::Excluded`].
61    pub excluded: u32,
62    /// Requirements with outcome [`Outcome::Unsupported`].
63    pub unsupported: u32,
64    /// Requirements with outcome [`Outcome::NotApplicable`].
65    pub not_applicable: u32,
66}
67
68/// One requirement's row in the report.
69#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
70#[non_exhaustive]
71pub struct RequirementReport {
72    /// The requirement ID (`AREA-NNN`).
73    pub id: String,
74    /// The requirement's RFC 2119 level, as registry text (`"MUST"`, …).
75    pub level: String,
76    /// The evaluation outcome.
77    pub outcome: Outcome,
78    /// Findings, in event order. Empty unless `outcome` is `fail` or `warn`.
79    #[serde(default, skip_serializing_if = "Vec::is_empty")]
80    pub findings: Vec<Finding>,
81    /// The documented exclusion reason, when `outcome` is `excluded`.
82    #[serde(default, skip_serializing_if = "Option::is_none")]
83    pub exclusion: Option<String>,
84    /// Check IDs the build lacks, when `outcome` is `unsupported`.
85    #[serde(default, skip_serializing_if = "Vec::is_empty")]
86    pub missing_checks: Vec<String>,
87    /// The undeclared capability gate, when `outcome` is `not-applicable`.
88    #[serde(default, skip_serializing_if = "Option::is_none")]
89    pub capability: Option<String>,
90}
91
92/// A complete validation report for one trace against one registry.
93#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
94#[non_exhaustive]
95pub struct Report {
96    /// The registry's protocol revision (`YYYY-MM-DD`).
97    pub revision: String,
98    /// Aggregate counts.
99    pub totals: Totals,
100    /// Per-requirement outcomes, in registry order.
101    pub requirements: Vec<RequirementReport>,
102}
103
104impl Report {
105    /// `true` when any requirement failed (errors, not warnings).
106    #[must_use]
107    pub const fn has_errors(&self) -> bool {
108        self.totals.fail > 0
109    }
110
111    /// `true` when any SHOULD-level requirement produced findings.
112    #[must_use]
113    pub const fn has_warnings(&self) -> bool {
114        self.totals.warn > 0
115    }
116
117    /// `true` when the registry referenced checks this build does not implement.
118    #[must_use]
119    pub const fn has_unsupported(&self) -> bool {
120        self.totals.unsupported > 0
121    }
122
123    /// Renders the human-readable form.
124    #[must_use]
125    pub fn render_human(&self) -> String {
126        let mut out = String::new();
127        let _ = writeln!(out, "MCP trace validation — revision {}", self.revision);
128        for row in &self.requirements {
129            let marker = match row.outcome {
130                Outcome::Pass => "PASS",
131                Outcome::Fail => "FAIL",
132                Outcome::Warn => "WARN",
133                Outcome::Excluded => "EXCL",
134                Outcome::Unsupported => "UNSUP",
135                Outcome::NotApplicable => "N/A",
136            };
137            let _ = writeln!(out, "  {marker:<5} {} ({})", row.id, row.level);
138            for finding in &row.findings {
139                match finding.seq {
140                    Some(seq) => {
141                        let _ = writeln!(out, "        seq {seq}: {}", finding.detail);
142                    }
143                    None => {
144                        let _ = writeln!(out, "        {}", finding.detail);
145                    }
146                }
147            }
148            if let Some(exclusion) = &row.exclusion {
149                let _ = writeln!(out, "        excluded: {exclusion}");
150            }
151            for check in &row.missing_checks {
152                let _ = writeln!(out, "        unsupported check: {check}");
153            }
154            if let Some(capability) = &row.capability {
155                let _ = writeln!(
156                    out,
157                    "        not applicable: capability {capability} was not declared in this session"
158                );
159            }
160        }
161        let totals = self.totals;
162        let _ = writeln!(
163            out,
164            "totals: {} pass, {} fail, {} warn, {} excluded, {} unsupported, {} not applicable",
165            totals.pass,
166            totals.fail,
167            totals.warn,
168            totals.excluded,
169            totals.unsupported,
170            totals.not_applicable
171        );
172        let _ = writeln!(out, "verdict: {}", self.verdict());
173        out
174    }
175
176    /// One-word verdict for the trailing summary line.
177    #[must_use]
178    pub const fn verdict(&self) -> Verdict {
179        if self.totals.unsupported > 0 {
180            Verdict::Unsupported
181        } else if self.totals.fail > 0 {
182            Verdict::Fail
183        } else if self.totals.warn > 0 {
184            Verdict::PassWithWarnings
185        } else {
186            Verdict::Pass
187        }
188    }
189}
190
191/// Overall verdict of a validation run.
192#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
193#[serde(rename_all = "kebab-case")]
194#[non_exhaustive]
195pub enum Verdict {
196    /// No findings at all.
197    Pass,
198    /// Only SHOULD-level findings.
199    PassWithWarnings,
200    /// At least one MUST-level violation.
201    Fail,
202    /// The registry and this build disagree about available checks.
203    Unsupported,
204}
205
206impl fmt::Display for Verdict {
207    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
208        let text = match self {
209            Self::Pass => "pass",
210            Self::PassWithWarnings => "pass-with-warnings",
211            Self::Fail => "fail",
212            Self::Unsupported => "unsupported",
213        };
214        f.write_str(text)
215    }
216}
217
218#[cfg(test)]
219#[allow(clippy::unwrap_used)]
220mod tests {
221    use super::*;
222
223    fn sample() -> Report {
224        Report {
225            revision: "2025-11-25".to_owned(),
226            totals: Totals {
227                pass: 1,
228                fail: 1,
229                warn: 0,
230                excluded: 1,
231                unsupported: 0,
232                not_applicable: 1,
233            },
234            requirements: vec![
235                RequirementReport {
236                    id: "BASE-001".to_owned(),
237                    level: "MUST".to_owned(),
238                    outcome: Outcome::Pass,
239                    findings: vec![],
240                    exclusion: None,
241                    missing_checks: vec![],
242                    capability: None,
243                },
244                RequirementReport {
245                    id: "LIFE-001".to_owned(),
246                    level: "MUST".to_owned(),
247                    outcome: Outcome::Fail,
248                    findings: vec![Finding {
249                        check: "lifecycle.first-interaction-initialize".to_owned(),
250                        seq: Some(3),
251                        detail: "first message is \"tools/list\", expected \"initialize\""
252                            .to_owned(),
253                    }],
254                    exclusion: None,
255                    missing_checks: vec![],
256                    capability: None,
257                },
258                RequirementReport {
259                    id: "TRAN-001".to_owned(),
260                    level: "MUST NOT".to_owned(),
261                    outcome: Outcome::Excluded,
262                    findings: vec![],
263                    exclusion: Some("enforced at capture time".to_owned()),
264                    missing_checks: vec![],
265                    capability: None,
266                },
267                RequirementReport {
268                    id: "TOOL-001".to_owned(),
269                    level: "MUST".to_owned(),
270                    outcome: Outcome::NotApplicable,
271                    findings: vec![],
272                    exclusion: None,
273                    missing_checks: vec![],
274                    capability: Some("server.tools".to_owned()),
275                },
276            ],
277        }
278    }
279
280    #[test]
281    fn verdict_priority_is_unsupported_fail_warn_pass() {
282        let mut report = sample();
283        assert_eq!(report.verdict(), Verdict::Fail);
284        report.totals.unsupported = 1;
285        assert_eq!(report.verdict(), Verdict::Unsupported);
286        report.totals.unsupported = 0;
287        report.totals.fail = 0;
288        report.totals.warn = 2;
289        assert_eq!(report.verdict(), Verdict::PassWithWarnings);
290        report.totals.warn = 0;
291        assert_eq!(report.verdict(), Verdict::Pass);
292    }
293
294    #[test]
295    fn human_rendering_shows_findings_and_totals() {
296        let text = sample().render_human();
297        assert!(text.contains("FAIL  LIFE-001 (MUST)"), "{text}");
298        assert!(text.contains("seq 3:"), "{text}");
299        assert!(
300            text.contains("excluded: enforced at capture time"),
301            "{text}"
302        );
303        assert!(text.contains("N/A   TOOL-001 (MUST)"), "{text}");
304        assert!(
305            text.contains(
306                "not applicable: capability server.tools was not declared in this session"
307            ),
308            "{text}"
309        );
310        assert!(
311            text.contains(
312                "totals: 1 pass, 1 fail, 0 warn, 1 excluded, 0 unsupported, 1 not applicable"
313            ),
314            "{text}"
315        );
316        assert!(text.contains("verdict: fail"), "{text}");
317    }
318
319    #[test]
320    fn json_omits_empty_collections() {
321        let report = sample();
322        let json = serde_json::to_string(&report).unwrap();
323        assert!(json.contains("\"revision\":\"2025-11-25\""), "{json}");
324        // Passing rows carry no findings/exclusion/missing_checks keys.
325        assert!(!json.contains("\"missing_checks\""), "{json}");
326    }
327
328    #[test]
329    fn totals_predicates_pin_their_thresholds() {
330        let mut report = sample();
331        report.totals = Totals::default();
332        assert!(!report.has_errors());
333        assert!(!report.has_warnings());
334        assert!(!report.has_unsupported());
335        report.totals.fail = 1;
336        assert!(report.has_errors());
337        report.totals.warn = 1;
338        assert!(report.has_warnings());
339        report.totals.unsupported = 1;
340        assert!(report.has_unsupported());
341    }
342}
mcp_trace_validator/report.rs

mcp_trace_validator/
report.rs
