eggress_testkit/
report.rs

1use std::fmt;
2use std::fs;
3use std::path::Path;
4use std::process::Command;
5use std::str::FromStr;
6
7use serde::{Deserialize, Serialize};
8
9/// Structured test outcome for parity reporting.
10///
11/// Distinguishes the reason for a skip so that reports cannot be
12/// mistaken for passing evidence when tests did not actually run.
13#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
14#[serde(rename_all = "snake_case")]
15pub enum TestStatus {
16    /// Test ran and passed.
17    Passed,
18    /// Test ran and failed.
19    Failed,
20    /// Test skipped because the env-var gate was not set (e.g. `EGRESS_REQUIRE_EXTERNAL_INTEROP`).
21    SkippedMissingGate,
22    /// Test skipped because the required external tool is not installed.
23    SkippedMissingTool,
24    /// Test was not run at all (e.g. CI job did not include this test group).
25    NotRun,
26}
27
28impl fmt::Display for TestStatus {
29    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
30        match self {
31            Self::Passed => write!(f, "passed"),
32            Self::Failed => write!(f, "failed"),
33            Self::SkippedMissingGate => write!(f, "skipped_missing_gate"),
34            Self::SkippedMissingTool => write!(f, "skipped_missing_tool"),
35            Self::NotRun => write!(f, "not_run"),
36        }
37    }
38}
39
40impl FromStr for TestStatus {
41    type Err = String;
42
43    fn from_str(s: &str) -> Result<Self, Self::Err> {
44        match s {
45            "passed" => Ok(Self::Passed),
46            "failed" => Ok(Self::Failed),
47            "skipped_missing_gate" => Ok(Self::SkippedMissingGate),
48            "skipped_missing_tool" => Ok(Self::SkippedMissingTool),
49            "not_run" => Ok(Self::NotRun),
50            other => Err(format!("unknown test status: \"{}\"", other)),
51        }
52    }
53}
54
55#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
56pub struct ParityReport {
57    pub eggress_commit: Option<String>,
58    pub pproxy_version: String,
59    pub os_platform: String,
60    pub rust_version: String,
61    pub python_version: String,
62    pub feature_gates: Vec<String>,
63    pub features: Vec<FeatureReport>,
64}
65
66#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
67pub struct FeatureReport {
68    pub feature_id: String,
69    pub category: String,
70    pub manifest_evidence: String,
71    pub tests_executed: Vec<String>,
72    pub status: TestStatus,
73    pub skip_reason: Option<String>,
74    pub observed_divergence: Option<String>,
75    pub suggested_evidence: String,
76}
77
78#[derive(Debug, Clone, Deserialize, PartialEq)]
79pub struct ManifestEntry {
80    pub feature_id: String,
81    pub category: String,
82    pub evidence: String,
83    #[serde(default)]
84    pub tests: Vec<String>,
85}
86
87impl Default for ParityReport {
88    fn default() -> Self {
89        Self::new()
90    }
91}
92
93impl ParityReport {
94    pub fn new() -> Self {
95        let rust_version = Command::new("rustc")
96            .arg("--version")
97            .output()
98            .ok()
99            .and_then(|o| {
100                if o.status.success() {
101                    String::from_utf8(o.stdout)
102                        .ok()
103                        .map(|s| s.trim().to_string())
104                } else {
105                    None
106                }
107            })
108            .unwrap_or_else(|| "unknown".to_string());
109
110        let python_version = detect_python_version().unwrap_or_else(|| "unknown".to_string());
111        let pproxy_version = detect_pproxy_version().unwrap_or_else(|| "unknown".to_string());
112
113        let os_platform = std::env::consts::OS.to_string();
114
115        Self {
116            eggress_commit: detect_eggress_commit(),
117            pproxy_version,
118            os_platform,
119            rust_version,
120            python_version,
121            feature_gates: Vec::new(),
122            features: Vec::new(),
123        }
124    }
125
126    pub fn add_feature(&mut self, feature: FeatureReport) {
127        self.features.push(feature);
128    }
129
130    pub fn to_json(&self) -> String {
131        serde_json::to_string_pretty(self).expect("serialization should not fail")
132    }
133
134    pub fn to_markdown(&self) -> String {
135        let mut md = String::new();
136
137        md.push_str("# Eggress Parity Report\n\n");
138
139        md.push_str("## Environment\n\n");
140        md.push_str("| Field | Value |\n");
141        md.push_str("|-------|-------|\n");
142        md.push_str(&format!(
143            "| eggress commit | {} |\n",
144            self.eggress_commit.as_deref().unwrap_or("n/a")
145        ));
146        md.push_str(&format!("| pproxy version | {} |\n", self.pproxy_version));
147        md.push_str(&format!("| OS platform | {} |\n", self.os_platform));
148        md.push_str(&format!("| rust version | {} |\n", self.rust_version));
149        md.push_str(&format!("| python version | {} |\n", self.python_version));
150
151        if !self.feature_gates.is_empty() {
152            md.push_str("\n## Feature Gates\n\n");
153            for gate in &self.feature_gates {
154                md.push_str(&format!("- `{}`\n", gate));
155            }
156        }
157
158        md.push_str("\n## Feature Results\n\n");
159        md.push_str("| Feature ID | Category | Status | Manifest Evidence | Tests Executed | Skip Reason | Divergence | Suggested Evidence |\n");
160        md.push_str("|-----------|----------|--------|-------------------|----------------|-------------|------------|--------------------|\n");
161
162        for f in &self.features {
163            let tests = f.tests_executed.join(", ");
164            let skip = f.skip_reason.as_deref().unwrap_or("-");
165            let divergence = f.observed_divergence.as_deref().unwrap_or("-");
166            md.push_str(&format!(
167                "| {} | {} | {} | {} | {} | {} | {} | {} |\n",
168                f.feature_id,
169                f.category,
170                f.status,
171                f.manifest_evidence,
172                tests,
173                skip,
174                divergence,
175                f.suggested_evidence,
176            ));
177        }
178
179        md
180    }
181
182    pub fn write_json(&self, path: &Path) -> std::io::Result<()> {
183        fs::write(path, self.to_json())
184    }
185
186    pub fn write_markdown(&self, path: &Path) -> std::io::Result<()> {
187        fs::write(path, self.to_markdown())
188    }
189}
190
191pub fn detect_eggress_commit() -> Option<String> {
192    if let Ok(val) = std::env::var("EGRESS_COMMIT") {
193        if !val.is_empty() {
194            return Some(val);
195        }
196    }
197
198    Command::new("git")
199        .args(["rev-parse", "--short", "HEAD"])
200        .output()
201        .ok()
202        .and_then(|o| {
203            if o.status.success() {
204                String::from_utf8(o.stdout)
205                    .ok()
206                    .map(|s| s.trim().to_string())
207            } else {
208                None
209            }
210        })
211}
212
213pub fn detect_python_version() -> Option<String> {
214    Command::new("python3")
215        .arg("--version")
216        .output()
217        .ok()
218        .and_then(|o| {
219            if o.status.success() {
220                String::from_utf8(o.stdout)
221                    .ok()
222                    .map(|s| s.trim().to_string())
223            } else {
224                None
225            }
226        })
227}
228
229pub fn detect_pproxy_version() -> Option<String> {
230    Command::new("python3")
231        .args(["-c", "import pproxy; print(pproxy.__version__)"])
232        .output()
233        .ok()
234        .and_then(|o| {
235            if o.status.success() {
236                String::from_utf8(o.stdout)
237                    .ok()
238                    .map(|s| s.trim().to_string())
239            } else {
240                None
241            }
242        })
243}
244
245#[derive(Debug, Deserialize)]
246struct ManifestFile {
247    #[serde(default)]
248    features: Vec<ManifestEntry>,
249}
250
251pub fn load_manifest(path: &Path) -> std::io::Result<Vec<ManifestEntry>> {
252    let content = fs::read_to_string(path)?;
253    let manifest: ManifestFile = toml::from_str(&content)
254        .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
255    Ok(manifest.features)
256}
257
258pub fn manifest_entry_to_feature(entry: &ManifestEntry) -> FeatureReport {
259    FeatureReport {
260        feature_id: entry.feature_id.clone(),
261        category: entry.category.clone(),
262        manifest_evidence: entry.evidence.clone(),
263        tests_executed: entry.tests.clone(),
264        status: TestStatus::NotRun,
265        skip_reason: None,
266        observed_divergence: None,
267        suggested_evidence: entry.evidence.clone(),
268    }
269}
270
271#[cfg(test)]
272mod tests {
273    use super::*;
274    use tempfile::NamedTempFile;
275
276    fn sample_report() -> ParityReport {
277        ParityReport {
278            eggress_commit: Some("abc1234".to_string()),
279            pproxy_version: "1.1.4".to_string(),
280            os_platform: "macos".to_string(),
281            rust_version: "rustc 1.75.0 (82e1608df 2023-12-21)".to_string(),
282            python_version: "Python 3.12.0".to_string(),
283            feature_gates: vec!["EGRESS_REQUIRE_SHADOWSOCKS_INTEROP".to_string()],
284            features: vec![
285                FeatureReport {
286                    feature_id: "socks5_tcp".to_string(),
287                    category: "protocol".to_string(),
288                    manifest_evidence: "SOCKS5 supported".to_string(),
289                    tests_executed: vec!["test_socks5_connect".to_string()],
290                    status: TestStatus::Passed,
291                    skip_reason: None,
292                    observed_divergence: None,
293                    suggested_evidence: "SOCKS5 supported".to_string(),
294                },
295                FeatureReport {
296                    feature_id: "shadowsocks_udp".to_string(),
297                    category: "protocol".to_string(),
298                    manifest_evidence: "SS UDP supported".to_string(),
299                    tests_executed: vec![],
300                    status: TestStatus::SkippedMissingTool,
301                    skip_reason: Some("pproxy not available".to_string()),
302                    observed_divergence: None,
303                    suggested_evidence: "SS UDP supported".to_string(),
304                },
305            ],
306        }
307    }
308
309    #[test]
310    fn json_round_trip() {
311        let report = sample_report();
312        let json = report.to_json();
313        let parsed: ParityReport = serde_json::from_str(&json).unwrap();
314        assert_eq!(parsed, report);
315    }
316
317    #[test]
318    fn markdown_generation() {
319        let report = sample_report();
320        let md = report.to_markdown();
321        assert!(md.contains("# Eggress Parity Report"));
322        assert!(md.contains("| eggress commit | abc1234 |"));
323        assert!(md.contains("| pproxy version | 1.1.4 |"));
324        assert!(md.contains("| rust version | rustc 1.75.0 (82e1608df 2023-12-21) |"));
325        assert!(md.contains("| python version | Python 3.12.0 |"));
326        assert!(md.contains("| socks5_tcp | protocol | passed |"));
327        assert!(md.contains("| shadowsocks_udp | protocol | skipped_missing_tool |"));
328        assert!(md.contains("EGRESS_REQUIRE_SHADOWSOCKS_INTEROP"));
329    }
330
331    #[test]
332    fn manifest_loading() {
333        let toml_content = r#"
334[[features]]
335feature_id = "test_feature"
336category = "test_cat"
337evidence = "test evidence"
338tests = ["test_a", "test_b"]
339"#;
340        let file = NamedTempFile::new().unwrap();
341        fs::write(file.path(), toml_content).unwrap();
342
343        let entries = load_manifest(file.path()).unwrap();
344        assert_eq!(entries.len(), 1);
345        assert_eq!(entries[0].feature_id, "test_feature");
346        assert_eq!(entries[0].category, "test_cat");
347        assert_eq!(entries[0].evidence, "test evidence");
348        assert_eq!(entries[0].tests, vec!["test_a", "test_b"]);
349    }
350
351    #[test]
352    fn manifest_entry_to_feature_conversion() {
353        let entry = ManifestEntry {
354            feature_id: "conv_test".to_string(),
355            category: "convert".to_string(),
356            evidence: "some evidence".to_string(),
357            tests: vec!["t1".to_string()],
358        };
359        let feature = manifest_entry_to_feature(&entry);
360        assert_eq!(feature.feature_id, "conv_test");
361        assert_eq!(feature.status, TestStatus::NotRun);
362        assert_eq!(feature.manifest_evidence, "some evidence");
363        assert_eq!(feature.suggested_evidence, "some evidence");
364        assert!(feature.skip_reason.is_none());
365    }
366
367    #[test]
368    fn manifest_loading_empty_array() {
369        let toml_content = "# empty\n";
370        let file = NamedTempFile::new().unwrap();
371        fs::write(file.path(), toml_content).unwrap();
372
373        let entries = load_manifest(file.path()).unwrap();
374        assert!(entries.is_empty());
375    }
376
377    #[test]
378    fn markdown_with_no_features() {
379        let report = ParityReport {
380            eggress_commit: None,
381            pproxy_version: "unknown".to_string(),
382            os_platform: "linux".to_string(),
383            rust_version: "unknown".to_string(),
384            python_version: "unknown".to_string(),
385            feature_gates: vec![],
386            features: vec![],
387        };
388        let md = report.to_markdown();
389        assert!(md.contains("| eggress commit | n/a |"));
390        assert!(md.contains("| pproxy version | unknown |"));
391        assert!(md.contains("| Feature ID | Category | Status"));
392    }
393
394    #[test]
395    fn write_json_to_file() {
396        let report = sample_report();
397        let file = NamedTempFile::new().unwrap();
398        report.write_json(file.path()).unwrap();
399
400        let content = fs::read_to_string(file.path()).unwrap();
401        let parsed: ParityReport = serde_json::from_str(&content).unwrap();
402        assert_eq!(parsed, report);
403    }
404
405    #[test]
406    fn write_markdown_to_file() {
407        let report = sample_report();
408        let file = NamedTempFile::new().unwrap();
409        report.write_markdown(file.path()).unwrap();
410
411        let content = fs::read_to_string(file.path()).unwrap();
412        assert!(content.contains("# Eggress Parity Report"));
413        assert!(content.contains("socks5_tcp"));
414    }
415
416    #[test]
417    fn test_status_roundtrip() {
418        for variant in &[
419            TestStatus::Passed,
420            TestStatus::Failed,
421            TestStatus::SkippedMissingGate,
422            TestStatus::SkippedMissingTool,
423            TestStatus::NotRun,
424        ] {
425            let s = variant.to_string();
426            let parsed: TestStatus = s.parse().unwrap();
427            assert_eq!(parsed, *variant);
428        }
429    }
430
431    #[test]
432    fn test_status_invalid_parse() {
433        let result = "bogus".parse::<TestStatus>();
434        assert!(result.is_err());
435    }
436
437    #[test]
438    fn test_status_serde() {
439        let status = TestStatus::SkippedMissingGate;
440        let json = serde_json::to_string(&status).unwrap();
441        assert_eq!(json, "\"skipped_missing_gate\"");
442        let parsed: TestStatus = serde_json::from_str(&json).unwrap();
443        assert_eq!(parsed, status);
444    }
445}
eggress_testkit/report.rs

eggress_testkit/
report.rs
