eggress_testkit/oracle/
report.rs

1//! Oracle JSON report generation.
2//!
3//! Produces structured comparison reports after running oracle scenarios.
4
5use std::collections::BTreeMap;
6use std::fs;
7use std::path::Path;
8use std::process::Command;
9use std::time::Duration;
10
11use serde::{Deserialize, Serialize};
12
13use super::observations::ProxyObservation;
14use super::scenario::ScenarioCategory;
15
16/// Certification profile for scenario filtering.
17#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
18#[serde(rename_all = "snake_case")]
19pub enum CertificationProfile {
20    /// Requires pproxy oracle and external interoperability.
21    Differential,
22    /// Platform-specific or privileged checks; explicitly selected.
23    Platform,
24}
25
26/// Top-level oracle report.
27#[derive(Debug, Clone, Serialize, Deserialize)]
28pub struct OracleReport {
29    /// pproxy version used.
30    pub pproxy_version: String,
31    /// eggress commit hash.
32    pub eggress_commit: Option<String>,
33    /// OS platform.
34    pub os_platform: String,
35    /// Rust version.
36    pub rust_version: String,
37    /// Python version.
38    pub python_version: String,
39    /// Total elapsed time.
40    pub elapsed_ms: u64,
41    /// Per-scenario results.
42    pub scenarios: Vec<ScenarioResult>,
43    /// Summary counts.
44    pub summary: ReportSummary,
45}
46
47/// Result of a single scenario.
48#[derive(Debug, Clone, Serialize, Deserialize)]
49pub struct ScenarioResult {
50    /// Scenario ID.
51    pub id: String,
52    /// Scenario category.
53    pub category: ScenarioCategory,
54    /// Scenario description.
55    pub description: String,
56    /// Outcome status.
57    pub status: ScenarioStatus,
58    /// Comparison results.
59    pub comparisons: Vec<ComparisonResult>,
60    /// Elapsed time for this scenario.
61    pub elapsed_ms: u64,
62    /// Error message if status is Error.
63    pub error: Option<String>,
64    /// Skip reason if status is Skipped.
65    pub skip_reason: Option<String>,
66    /// Observations from pproxy execution.
67    #[serde(skip_serializing_if = "Option::is_none")]
68    pub pproxy_observation: Option<ProxyObservation>,
69    /// Observations from eggress execution.
70    #[serde(skip_serializing_if = "Option::is_none")]
71    pub eggress_observation: Option<ProxyObservation>,
72    /// Timing tolerance in milliseconds.
73    #[serde(skip_serializing_if = "Option::is_none")]
74    pub timing_tolerance_ms: Option<u64>,
75    /// Divergence IDs.
76    #[serde(default, skip_serializing_if = "Vec::is_empty")]
77    pub divergence_ids: Vec<String>,
78    /// Certification profile.
79    #[serde(rename = "profile", skip_serializing_if = "Option::is_none")]
80    pub certification_profile: Option<CertificationProfile>,
81    /// Capability IDs from the scenario definition.
82    #[serde(default, skip_serializing_if = "Vec::is_empty")]
83    pub capability_ids: Vec<String>,
84}
85
86/// Status of a scenario execution.
87#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
88#[serde(rename_all = "snake_case")]
89pub enum ScenarioStatus {
90    Pass,
91    Fail,
92    Skipped,
93    Error,
94}
95
96/// A single comparison within a scenario.
97#[derive(Debug, Clone, Serialize, Deserialize)]
98pub struct ComparisonResult {
99    /// What is being compared (e.g., "tcp_echo_payload", "exit_code").
100    pub dimension: String,
101    /// pproxy result (serialized).
102    pub pproxy_value: String,
103    /// eggress result (serialized).
104    pub eggress_value: String,
105    /// Whether the comparison matched.
106    pub matched: bool,
107    /// Details if mismatched.
108    pub details: Option<String>,
109}
110
111/// Summary counts for the report.
112#[derive(Debug, Clone, Serialize, Deserialize)]
113pub struct ReportSummary {
114    pub total: usize,
115    pub passed: usize,
116    pub failed: usize,
117    pub skipped: usize,
118    pub errors: usize,
119}
120
121impl Default for OracleReport {
122    fn default() -> Self {
123        Self::new()
124    }
125}
126
127impl OracleReport {
128    /// Create a new empty report with environment metadata.
129    pub fn new() -> Self {
130        Self {
131            pproxy_version: detect_pproxy_version().unwrap_or_else(|| "unknown".to_string()),
132            eggress_commit: detect_eggress_commit(),
133            os_platform: std::env::consts::OS.to_string(),
134            rust_version: detect_rust_version(),
135            python_version: detect_python_version().unwrap_or_else(|| "unknown".to_string()),
136            elapsed_ms: 0,
137            scenarios: Vec::new(),
138            summary: ReportSummary {
139                total: 0,
140                passed: 0,
141                failed: 0,
142                skipped: 0,
143                errors: 0,
144            },
145        }
146    }
147
148    /// Add a scenario result and update summary.
149    pub fn add_scenario(&mut self, result: ScenarioResult) {
150        match result.status {
151            ScenarioStatus::Pass => self.summary.passed += 1,
152            ScenarioStatus::Fail => self.summary.failed += 1,
153            ScenarioStatus::Skipped => self.summary.skipped += 1,
154            ScenarioStatus::Error => self.summary.errors += 1,
155        }
156        self.summary.total += 1;
157        self.scenarios.push(result);
158    }
159
160    /// Set total elapsed time.
161    pub fn set_elapsed(&mut self, elapsed: Duration) {
162        self.elapsed_ms = elapsed.as_millis() as u64;
163    }
164
165    /// Serialize to pretty JSON.
166    pub fn to_json(&self) -> String {
167        serde_json::to_string_pretty(self).expect("oracle report serialization should not fail")
168    }
169
170    /// Write report to a JSON file.
171    pub fn write_json(&self, path: &Path) -> std::io::Result<()> {
172        if let Some(parent) = path.parent() {
173            fs::create_dir_all(parent)?;
174        }
175        fs::write(path, self.to_json())
176    }
177
178    /// Generate a human-readable Markdown report.
179    pub fn to_markdown(&self) -> String {
180        let mut md = String::new();
181        md.push_str("# Oracle Differential Report\n\n");
182        md.push_str(&format!("**pproxy version:** {}\n", self.pproxy_version));
183        md.push_str(&format!(
184            "**eggress commit:** {}\n",
185            self.eggress_commit.as_deref().unwrap_or("unknown")
186        ));
187        md.push_str(&format!("**platform:** {}\n", self.os_platform));
188        md.push_str(&format!("**elapsed:** {}ms\n\n", self.elapsed_ms));
189
190        md.push_str("## Summary\n\n");
191        md.push_str("| Metric | Count |\n|--------|-------|\n");
192        md.push_str(&format!("| Total | {} |\n", self.summary.total));
193        md.push_str(&format!("| Passed | {} |\n", self.summary.passed));
194        md.push_str(&format!("| Failed | {} |\n", self.summary.failed));
195        md.push_str(&format!("| Skipped | {} |\n", self.summary.skipped));
196        md.push_str(&format!("| Errors | {} |\n\n", self.summary.errors));
197
198        let mut by_category: BTreeMap<String, Vec<&ScenarioResult>> = BTreeMap::new();
199        for s in &self.scenarios {
200            let cat = format!("{:?}", s.category);
201            by_category.entry(cat).or_default().push(s);
202        }
203
204        for (cat, scenarios) in &by_category {
205            md.push_str(&format!("## {}\n\n", cat));
206            for s in scenarios {
207                let status_icon = match s.status {
208                    ScenarioStatus::Pass => "✅",
209                    ScenarioStatus::Fail => "❌",
210                    ScenarioStatus::Skipped => "⏭️",
211                    ScenarioStatus::Error => "⚠️",
212                };
213                md.push_str(&format!(
214                    "### {} {} {}\n\n",
215                    status_icon, s.id, s.description
216                ));
217
218                if let Some(err) = &s.error {
219                    md.push_str(&format!("**Error:** {}\n\n", err));
220                }
221                if let Some(skip) = &s.skip_reason {
222                    md.push_str(&format!("**Skip reason:** {}\n\n", skip));
223                }
224
225                if !s.comparisons.is_empty() {
226                    md.push_str("| Dimension | Match | Pproxy | Eggress |\n");
227                    md.push_str("|-----------|-------|--------|----------|\n");
228                    for c in &s.comparisons {
229                        let match_icon = if c.matched { "✅" } else { "❌" };
230                        md.push_str(&format!(
231                            "| {} | {} | {} | {} |\n",
232                            c.dimension,
233                            match_icon,
234                            truncate_md(&c.pproxy_value, 50),
235                            truncate_md(&c.eggress_value, 50)
236                        ));
237                    }
238                    md.push('\n');
239                }
240
241                if !s.divergence_ids.is_empty() {
242                    md.push_str(&format!(
243                        "**Divergences:** {}\n\n",
244                        s.divergence_ids.join(", ")
245                    ));
246                }
247            }
248        }
249
250        md
251    }
252
253    /// Check manifest consistency: verify every scenario references valid capability IDs.
254    pub fn check_manifest_consistency(&self, valid_capability_ids: &[&str]) -> Vec<String> {
255        let mut warnings = Vec::new();
256        for s in &self.scenarios {
257            for cap_id in &s.capability_ids {
258                if !valid_capability_ids.contains(&cap_id.as_str()) {
259                    warnings.push(format!(
260                        "scenario '{}' references unknown capability '{}'",
261                        s.id, cap_id
262                    ));
263                }
264            }
265        }
266        warnings
267    }
268
269    /// Filter scenarios by certification profile.
270    pub fn scenarios_for_profile(&self, profile: CertificationProfile) -> Vec<&ScenarioResult> {
271        self.scenarios
272            .iter()
273            .filter(|s| s.certification_profile == Some(profile))
274            .collect()
275    }
276}
277
278impl ScenarioResult {
279    pub fn new(id: &str, category: ScenarioCategory, description: &str) -> Self {
280        Self {
281            id: id.to_string(),
282            category,
283            description: description.to_string(),
284            status: ScenarioStatus::Skipped,
285            comparisons: Vec::new(),
286            elapsed_ms: 0,
287            error: None,
288            skip_reason: None,
289            pproxy_observation: None,
290            eggress_observation: None,
291            timing_tolerance_ms: None,
292            divergence_ids: Vec::new(),
293            certification_profile: None,
294            capability_ids: Vec::new(),
295        }
296    }
297
298    pub fn with_status(mut self, status: ScenarioStatus) -> Self {
299        self.status = status;
300        self
301    }
302
303    pub fn with_elapsed(mut self, elapsed: Duration) -> Self {
304        self.elapsed_ms = elapsed.as_millis() as u64;
305        self
306    }
307
308    pub fn with_comparisons(mut self, comparisons: Vec<ComparisonResult>) -> Self {
309        self.comparisons = comparisons;
310        self
311    }
312
313    pub fn with_error(mut self, error: String) -> Self {
314        self.error = Some(error);
315        self
316    }
317
318    pub fn with_skip_reason(mut self, reason: String) -> Self {
319        self.skip_reason = Some(reason);
320        self.status = ScenarioStatus::Skipped;
321        self
322    }
323
324    pub fn with_pproxy_observation(mut self, obs: ProxyObservation) -> Self {
325        self.pproxy_observation = Some(obs);
326        self
327    }
328
329    pub fn with_eggress_observation(mut self, obs: ProxyObservation) -> Self {
330        self.eggress_observation = Some(obs);
331        self
332    }
333
334    pub fn with_timing_tolerance(mut self, tolerance_ms: u64) -> Self {
335        self.timing_tolerance_ms = Some(tolerance_ms);
336        self
337    }
338
339    pub fn with_divergences(mut self, ids: Vec<String>) -> Self {
340        self.divergence_ids = ids;
341        self
342    }
343
344    pub fn with_certification_profile(mut self, profile: CertificationProfile) -> Self {
345        self.certification_profile = Some(profile);
346        self
347    }
348
349    pub fn with_capability_ids(mut self, ids: Vec<String>) -> Self {
350        self.capability_ids = ids;
351        self
352    }
353}
354
355fn truncate_md(s: &str, max: usize) -> String {
356    if s.len() <= max {
357        s.replace('|', "\\|")
358    } else {
359        format!("{}... ({} bytes)", s[..max].replace('|', "\\|"), s.len())
360    }
361}
362
363/// Port pattern shared by [`normalize_for_comparison`], compiled once.
364fn port_substitution_regex() -> &'static regex::Regex {
365    static PORT_RE: std::sync::OnceLock<regex::Regex> = std::sync::OnceLock::new();
366    PORT_RE.get_or_init(|| regex::Regex::new(r":\d{4,5}").expect("valid port regex"))
367}
368
369/// Normalize a value for comparison (strip ports, versions, etc.).
370pub fn normalize_for_comparison(value: &str, scenario_id: &str) -> String {
371    let mut result = value.to_string();
372
373    result = port_substitution_regex()
374        .replace_all(&result, ":PORT")
375        .into_owned();
376
377    if scenario_id.starts_with("cli.") {
378        for prefix in &["INFO:", "WARNING:", "DEBUG:", "Listen: "] {
379            result = result.replace(prefix, "");
380        }
381    }
382
383    result = result.replace("pproxy/", "");
384    result = result.replace("eggress/", "");
385
386    result.trim().to_string()
387}
388
389/// Create a comparison result.
390pub fn make_comparison(
391    dimension: &str,
392    pproxy_value: &str,
393    eggress_value: &str,
394) -> ComparisonResult {
395    let matched = pproxy_value == eggress_value;
396    let details = if matched {
397        None
398    } else {
399        Some(format!(
400            "pproxy: {}, eggress: {}",
401            truncate(pproxy_value, 200),
402            truncate(eggress_value, 200)
403        ))
404    };
405    ComparisonResult {
406        dimension: dimension.to_string(),
407        pproxy_value: pproxy_value.to_string(),
408        eggress_value: eggress_value.to_string(),
409        matched,
410        details,
411    }
412}
413
414fn truncate(s: &str, max_len: usize) -> String {
415    if s.len() <= max_len {
416        s.to_string()
417    } else {
418        format!("{}... ({} bytes total)", &s[..max_len], s.len())
419    }
420}
421
422fn detect_pproxy_version() -> Option<String> {
423    let python = match std::panic::catch_unwind(crate::differential::find_python_binary) {
424        Ok(p) => p,
425        Err(_) => return None,
426    };
427    Command::new(&python)
428        .args([
429            "-c",
430            "import pproxy; print(getattr(pproxy, '__version__', 'unknown'))",
431        ])
432        .stdout(std::process::Stdio::piped())
433        .stderr(std::process::Stdio::null())
434        .output()
435        .ok()
436        .and_then(|o| {
437            if o.status.success() {
438                String::from_utf8(o.stdout)
439                    .ok()
440                    .map(|s| s.trim().to_string())
441            } else {
442                None
443            }
444        })
445}
446
447fn detect_eggress_commit() -> Option<String> {
448    if let Ok(val) = std::env::var("EGRESS_COMMIT") {
449        if !val.is_empty() {
450            return Some(val);
451        }
452    }
453    Command::new("git")
454        .args(["rev-parse", "--short", "HEAD"])
455        .output()
456        .ok()
457        .and_then(|o| {
458            if o.status.success() {
459                String::from_utf8(o.stdout)
460                    .ok()
461                    .map(|s| s.trim().to_string())
462            } else {
463                None
464            }
465        })
466}
467
468fn detect_python_version() -> Option<String> {
469    Command::new("python3")
470        .arg("--version")
471        .output()
472        .ok()
473        .and_then(|o| {
474            if o.status.success() {
475                String::from_utf8(o.stdout)
476                    .ok()
477                    .map(|s| s.trim().to_string())
478            } else {
479                None
480            }
481        })
482}
483
484fn detect_rust_version() -> String {
485    Command::new("rustc")
486        .arg("--version")
487        .output()
488        .ok()
489        .and_then(|o| {
490            if o.status.success() {
491                String::from_utf8(o.stdout)
492                    .ok()
493                    .map(|s| s.trim().to_string())
494            } else {
495                None
496            }
497        })
498        .unwrap_or_else(|| "unknown".to_string())
499}
500
501#[cfg(test)]
502mod tests {
503    use super::*;
504
505    #[test]
506    fn report_json_roundtrip() {
507        let mut report = OracleReport::new();
508        report.add_scenario(
509            ScenarioResult::new("test", ScenarioCategory::CliDefaults, "test scenario")
510                .with_status(ScenarioStatus::Pass)
511                .with_elapsed(Duration::from_millis(100)),
512        );
513        report.set_elapsed(Duration::from_secs(5));
514
515        let json = report.to_json();
516        let parsed: OracleReport = serde_json::from_str(&json).unwrap();
517        assert_eq!(parsed.scenarios.len(), 1);
518        assert_eq!(parsed.summary.total, 1);
519        assert_eq!(parsed.summary.passed, 1);
520        assert_eq!(parsed.elapsed_ms, 5000);
521    }
522
523    #[test]
524    fn make_comparison_match() {
525        let comp = make_comparison("payload", "hello", "hello");
526        assert!(comp.matched);
527        assert!(comp.details.is_none());
528    }
529
530    #[test]
531    fn make_comparison_mismatch() {
532        let comp = make_comparison("payload", "hello", "world");
533        assert!(!comp.matched);
534        assert!(comp.details.is_some());
535    }
536
537    #[test]
538    fn scenario_category_serde() {
539        let cat = ScenarioCategory::CliDefaults;
540        let json = serde_json::to_string(&cat).unwrap();
541        assert_eq!(json, "\"cli_defaults\"");
542    }
543
544    #[test]
545    fn markdown_report_generation() {
546        let mut report = OracleReport::new();
547        report.add_scenario(
548            ScenarioResult::new("test.pass", ScenarioCategory::CliDefaults, "passing test")
549                .with_status(ScenarioStatus::Pass)
550                .with_comparisons(vec![make_comparison("payload", "hello", "hello")]),
551        );
552        report.add_scenario(
553            ScenarioResult::new("test.fail", ScenarioCategory::HttpSocksTcp, "failing test")
554                .with_status(ScenarioStatus::Fail)
555                .with_error("mismatch".to_string()),
556        );
557
558        let md = report.to_markdown();
559        assert!(md.contains("# Oracle Differential Report"));
560        assert!(md.contains("test.pass"));
561        assert!(md.contains("test.fail"));
562        assert!(md.contains("✅"));
563        assert!(md.contains("❌"));
564    }
565
566    #[test]
567    fn manifest_consistency_check() {
568        let mut report = OracleReport::new();
569        report.add_scenario(
570            ScenarioResult::new("test", ScenarioCategory::CliDefaults, "test")
571                .with_capability_ids(vec!["valid.cap".to_string(), "unknown.cap".to_string()]),
572        );
573
574        let warnings = report.check_manifest_consistency(&["valid.cap"]);
575        assert_eq!(warnings.len(), 1);
576        assert!(warnings[0].contains("unknown.cap"));
577    }
578
579    #[test]
580    fn manifest_consistency_no_warnings_for_valid() {
581        let mut report = OracleReport::new();
582        report.add_scenario(
583            ScenarioResult::new("test", ScenarioCategory::CliDefaults, "test")
584                .with_capability_ids(vec!["cap.a".to_string(), "cap.b".to_string()]),
585        );
586
587        let warnings = report.check_manifest_consistency(&["cap.a", "cap.b"]);
588        assert!(warnings.is_empty());
589    }
590
591    #[test]
592    fn tier_filtering() {
593        let mut report = OracleReport::new();
594        report.add_scenario(
595            ScenarioResult::new("fast", ScenarioCategory::CliDefaults, "fast")
596                .with_certification_profile(CertificationProfile::Differential),
597        );
598        report.add_scenario(
599            ScenarioResult::new("core", ScenarioCategory::HttpSocksTcp, "core")
600                .with_certification_profile(CertificationProfile::Differential),
601        );
602        report.add_scenario(ScenarioResult::new(
603            "no_profile",
604            ScenarioCategory::Chains,
605            "no profile",
606        ));
607
608        let differential = report.scenarios_for_profile(CertificationProfile::Differential);
609        assert_eq!(differential.len(), 2);
610
611        let platform = report.scenarios_for_profile(CertificationProfile::Platform);
612        assert!(platform.is_empty());
613    }
614
615    #[test]
616    fn scenario_result_builder() {
617        let result = ScenarioResult::new("id", ScenarioCategory::Chains, "desc")
618            .with_status(ScenarioStatus::Pass)
619            .with_elapsed(Duration::from_secs(1))
620            .with_comparisons(vec![make_comparison("dim", "a", "a")])
621            .with_divergences(vec!["div1".to_string()])
622            .with_certification_profile(CertificationProfile::Differential)
623            .with_capability_ids(vec!["cap1".to_string()]);
624
625        assert_eq!(result.status, ScenarioStatus::Pass);
626        assert_eq!(result.elapsed_ms, 1000);
627        assert_eq!(result.divergence_ids, vec!["div1"]);
628        assert_eq!(
629            result.certification_profile,
630            Some(CertificationProfile::Differential)
631        );
632        assert_eq!(result.capability_ids, vec!["cap1"]);
633    }
634
635    #[test]
636    fn scenario_result_builder_skip_reason_sets_status() {
637        let result = ScenarioResult::new("id", ScenarioCategory::CliDefaults, "desc")
638            .with_status(ScenarioStatus::Pass)
639            .with_skip_reason("missing deps".to_string());
640        assert_eq!(result.status, ScenarioStatus::Skipped);
641    }
642
643    #[test]
644    fn certification_profile_serde() {
645        let profile = CertificationProfile::Differential;
646        let json = serde_json::to_string(&profile).unwrap();
647        assert_eq!(json, "\"differential\"");
648        let parsed: CertificationProfile = serde_json::from_str(&json).unwrap();
649        assert_eq!(parsed, profile);
650    }
651
652    #[test]
653    fn markdown_includes_divergences() {
654        let mut report = OracleReport::new();
655        report.add_scenario(
656            ScenarioResult::new("div_test", ScenarioCategory::Udp, "udp test")
657                .with_status(ScenarioStatus::Pass)
658                .with_divergences(vec!["div.timing".to_string(), "div.payload".to_string()]),
659        );
660
661        let md = report.to_markdown();
662        assert!(md.contains("**Divergences:** div.timing, div.payload"));
663    }
664
665    #[test]
666    fn markdown_includes_skip_reason() {
667        let mut report = OracleReport::new();
668        report.add_scenario(
669            ScenarioResult::new("skip_test", ScenarioCategory::CliDefaults, "skipped")
670                .with_skip_reason("needs root".to_string()),
671        );
672
673        let md = report.to_markdown();
674        assert!(md.contains("**Skip reason:** needs root"));
675        assert!(md.contains("⏭️"));
676    }
677
678    #[test]
679    fn markdown_truncates_long_values() {
680        let mut report = OracleReport::new();
681        let long_value = "x".repeat(200);
682        report.add_scenario(
683            ScenarioResult::new("long", ScenarioCategory::HttpSocksTcp, "long values")
684                .with_status(ScenarioStatus::Fail)
685                .with_comparisons(vec![make_comparison("dim", &long_value, "short")]),
686        );
687
688        let md = report.to_markdown();
689        assert!(md.contains("... (200 bytes)"));
690    }
691
692    #[test]
693    fn json_roundtrip_preserves_new_fields() {
694        let mut report = OracleReport::new();
695        report.add_scenario(
696            ScenarioResult::new("full", ScenarioCategory::Chains, "full fields")
697                .with_status(ScenarioStatus::Pass)
698                .with_timing_tolerance(50)
699                .with_divergences(vec!["d1".to_string()])
700                .with_certification_profile(CertificationProfile::Differential)
701                .with_capability_ids(vec!["cap.x".to_string()]),
702        );
703
704        let json = report.to_json();
705        let parsed: OracleReport = serde_json::from_str(&json).unwrap();
706        let s = &parsed.scenarios[0];
707        assert_eq!(s.timing_tolerance_ms, Some(50));
708        assert_eq!(s.divergence_ids, vec!["d1"]);
709        assert_eq!(
710            s.certification_profile,
711            Some(CertificationProfile::Differential)
712        );
713        assert_eq!(s.capability_ids, vec!["cap.x"]);
714    }
715}
eggress_testkit/oracle/report.rs

eggress_testkit/oracle/
report.rs
