jellyflow_runtime/runtime/conformance/
reports.rs

1use std::fmt;
2
3use serde::{Deserialize, Serialize};
4
5use super::capability::{ConformanceCapabilityGap, ConformanceCapabilityMatrix};
6use super::scenario::ConformanceTraceEvent;
7
8#[derive(Debug, Clone, Serialize, Deserialize)]
9pub struct ConformanceRunReport {
10    pub scenario: String,
11    pub actual_trace: Vec<ConformanceTraceEvent>,
12    #[serde(default, skip_serializing_if = "Vec::is_empty")]
13    pub mismatches: Vec<ConformanceTraceMismatch>,
14}
15
16impl ConformanceRunReport {
17    pub fn new(
18        scenario: impl Into<String>,
19        actual_trace: Vec<ConformanceTraceEvent>,
20        expected_trace: &[ConformanceTraceEvent],
21    ) -> Self {
22        let mismatches = trace_mismatches(expected_trace, &actual_trace);
23        Self {
24            scenario: scenario.into(),
25            actual_trace,
26            mismatches,
27        }
28    }
29
30    pub fn is_match(&self) -> bool {
31        self.mismatches.is_empty()
32    }
33
34    pub fn actual_trace(&self) -> &[ConformanceTraceEvent] {
35        &self.actual_trace
36    }
37
38    pub fn mismatches(&self) -> &[ConformanceTraceMismatch] {
39        &self.mismatches
40    }
41}
42
43impl fmt::Display for ConformanceRunReport {
44    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
45        if self.is_match() {
46            return write!(
47                f,
48                "conformance scenario `{}` matched {} trace events",
49                self.scenario,
50                self.actual_trace.len()
51            );
52        }
53
54        writeln!(
55            f,
56            "conformance trace mismatch for scenario `{}` ({} mismatch(es))",
57            self.scenario,
58            self.mismatches.len()
59        )?;
60        for mismatch in self.mismatches.iter().take(8) {
61            writeln!(
62                f,
63                "  [{}] expected: {:?}; actual: {:?}",
64                mismatch.index, mismatch.expected, mismatch.actual
65            )?;
66        }
67        if self.mismatches.len() > 8 {
68            writeln!(f, "  ... {} more", self.mismatches.len() - 8)?;
69        }
70        Ok(())
71    }
72}
73
74#[derive(Debug, Clone, Serialize, Deserialize)]
75pub struct ConformanceSuiteReport {
76    pub suite: String,
77    #[serde(default, skip_serializing_if = "ConformanceCapabilityMatrix::is_empty")]
78    pub capabilities: ConformanceCapabilityMatrix,
79    #[serde(default, skip_serializing_if = "Vec::is_empty")]
80    pub capability_gaps: Vec<ConformanceCapabilityGap>,
81    pub scenario_reports: Vec<ConformanceRunReport>,
82    #[serde(default, skip_serializing_if = "Vec::is_empty")]
83    pub errors: Vec<ConformanceRunError>,
84}
85
86impl ConformanceSuiteReport {
87    pub fn is_match(&self) -> bool {
88        self.errors.is_empty()
89            && self.capability_gaps.is_empty()
90            && self
91                .scenario_reports
92                .iter()
93                .all(ConformanceRunReport::is_match)
94    }
95
96    pub fn failed_scenarios(&self) -> usize {
97        self.errors.len()
98            + self
99                .scenario_reports
100                .iter()
101                .filter(|report| !report.is_match())
102                .count()
103    }
104
105    pub fn scenario_count(&self) -> usize {
106        self.scenario_reports.len() + self.errors.len()
107    }
108}
109
110impl fmt::Display for ConformanceSuiteReport {
111    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
112        if self.is_match() {
113            write!(
114                f,
115                "conformance suite `{}` matched {} scenario(s)",
116                self.suite,
117                self.scenario_count()
118            )?;
119            if let Some(adapter) = self.capabilities.adapter.as_deref() {
120                write!(f, " for adapter `{adapter}`")?;
121            }
122            return Ok(());
123        }
124
125        writeln!(
126            f,
127            "conformance suite `{}` failed: {} scenario(s), {} execution error(s), {} capability gap(s)",
128            self.suite,
129            self.failed_scenarios(),
130            self.errors.len(),
131            self.capability_gaps.len()
132        )?;
133        for gap in self.capability_gaps.iter().take(8) {
134            writeln!(
135                f,
136                "  capability {:?} required {:?}, actual {:?}",
137                gap.capability, gap.required, gap.actual
138            )?;
139        }
140        for report in self
141            .scenario_reports
142            .iter()
143            .filter(|report| !report.is_match())
144            .take(8)
145        {
146            writeln!(
147                f,
148                "  scenario `{}` mismatched {} trace event(s)",
149                report.scenario,
150                report.mismatches.len()
151            )?;
152        }
153        for error in self.errors.iter().take(8) {
154            writeln!(
155                f,
156                "  scenario `{}` errored at action {} ({}): {}",
157                error.scenario, error.action_index, error.action_kind, error.message
158            )?;
159        }
160        Ok(())
161    }
162}
163
164#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
165pub struct ConformanceTraceMismatch {
166    pub index: usize,
167    pub expected: Option<ConformanceTraceEvent>,
168    pub actual: Option<ConformanceTraceEvent>,
169}
170
171#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, thiserror::Error)]
172#[error(
173    "conformance scenario `{scenario}` failed at action {action_index} ({action_kind}): {message}"
174)]
175pub struct ConformanceRunError {
176    pub scenario: String,
177    pub action_index: usize,
178    pub action_kind: String,
179    pub message: String,
180}
181
182fn trace_mismatches(
183    expected: &[ConformanceTraceEvent],
184    actual: &[ConformanceTraceEvent],
185) -> Vec<ConformanceTraceMismatch> {
186    let len = expected.len().max(actual.len());
187    (0..len)
188        .filter_map(|index| {
189            let expected = expected.get(index);
190            let actual = actual.get(index);
191            (expected != actual).then(|| ConformanceTraceMismatch {
192                index,
193                expected: expected.cloned(),
194                actual: actual.cloned(),
195            })
196        })
197        .collect()
198}
jellyflow_runtime/runtime/conformance/reports.rs

jellyflow_runtime/runtime/conformance/
reports.rs
