eggress_testkit/oracle/
report.rs

1//! Oracle JSON report generation.
2//!
3//! Produces structured comparison reports after running oracle scenarios.
4
5use std::collections::BTreeMap;
6use std::fs;
7use std::path::Path;
8use std::process::Command;
9use std::time::Duration;
10
11use serde::{Deserialize, Serialize};
12
13use super::observations::ProxyObservation;
14use super::scenario::ScenarioCategory;
15
16/// Certification profile for scenario filtering.
17#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
18#[serde(rename_all = "snake_case")]
19pub enum CertificationProfile {
20    /// Requires pproxy oracle and external interoperability.
21    Differential,
22    /// Platform-specific or privileged checks; explicitly selected.
23    Platform,
24}
25
26/// Top-level oracle report.
27#[derive(Debug, Clone, Serialize, Deserialize)]
28pub struct OracleReport {
29    /// pproxy version used.
30    pub pproxy_version: String,
31    /// eggress commit hash.
32    pub eggress_commit: Option<String>,
33    /// OS platform.
34    pub os_platform: String,
35    /// Rust version.
36    pub rust_version: String,
37    /// Python version.
38    pub python_version: String,
39    /// Total elapsed time.
40    pub elapsed_ms: u64,
41    /// Per-scenario results.
42    pub scenarios: Vec<ScenarioResult>,
43    /// Summary counts.
44    pub summary: ReportSummary,
45}
46
47/// Result of a single scenario.
48#[derive(Debug, Clone, Serialize, Deserialize)]
49pub struct ScenarioResult {
50    /// Scenario ID.
51    pub id: String,
52    /// Scenario category.
53    pub category: ScenarioCategory,
54    /// Scenario description.
55    pub description: String,
56    /// Outcome status.
57    pub status: ScenarioStatus,
58    /// Comparison results.
59    pub comparisons: Vec<ComparisonResult>,
60    /// Elapsed time for this scenario.
61    pub elapsed_ms: u64,
62    /// Error message if status is Error.
63    pub error: Option<String>,
64    /// Skip reason if status is Skipped.
65    pub skip_reason: Option<String>,
66    /// Observations from pproxy execution.
67    #[serde(skip_serializing_if = "Option::is_none")]
68    pub pproxy_observation: Option<ProxyObservation>,
69    /// Observations from eggress execution.
70    #[serde(skip_serializing_if = "Option::is_none")]
71    pub eggress_observation: Option<ProxyObservation>,
72    /// Timing tolerance in milliseconds.
73    #[serde(skip_serializing_if = "Option::is_none")]
74    pub timing_tolerance_ms: Option<u64>,
75    /// Divergence IDs.
76    #[serde(default, skip_serializing_if = "Vec::is_empty")]
77    pub divergence_ids: Vec<String>,
78    /// Certification profile.
79    #[serde(rename = "profile", skip_serializing_if = "Option::is_none")]
80    pub certification_profile: Option<CertificationProfile>,
81    /// Capability IDs from the scenario definition.
82    #[serde(default, skip_serializing_if = "Vec::is_empty")]
83    pub capability_ids: Vec<String>,
84}
85
86/// Status of a scenario execution.
87#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
88#[serde(rename_all = "snake_case")]
89pub enum ScenarioStatus {
90    Pass,
91    Fail,
92    Skipped,
93    Error,
94}
95
96/// A single comparison within a scenario.
97#[derive(Debug, Clone, Serialize, Deserialize)]
98pub struct ComparisonResult {
99    /// What is being compared (e.g., "tcp_echo_payload", "exit_code").
100    pub dimension: String,
101    /// pproxy result (serialized).
102    pub pproxy_value: String,
103    /// eggress result (serialized).
104    pub eggress_value: String,
105    /// Whether the comparison matched.
106    pub matched: bool,
107    /// Details if mismatched.
108    pub details: Option<String>,
109}
110
111/// Summary counts for the report.
112#[derive(Debug, Clone, Serialize, Deserialize)]
113pub struct ReportSummary {
114    pub total: usize,
115    pub passed: usize,
116    pub failed: usize,
117    pub skipped: usize,
118    pub errors: usize,
119}
120
121impl Default for OracleReport {
122    fn default() -> Self {
123        Self::new()
124    }
125}
126
127impl OracleReport {
128    /// Create a new empty report with environment metadata.
129    pub fn new() -> Self {
130        Self {
131            pproxy_version: detect_pproxy_version().unwrap_or_else(|| "unknown".to_string()),
132            eggress_commit: detect_eggress_commit(),
133            os_platform: std::env::consts::OS.to_string(),
134            rust_version: detect_rust_version(),
135            python_version: detect_python_version().unwrap_or_else(|| "unknown".to_string()),
136            elapsed_ms: 0,
137            scenarios: Vec::new(),
138            summary: ReportSummary {
139                total: 0,
140                passed: 0,
141                failed: 0,
142                skipped: 0,
143                errors: 0,
144            },
145        }
146    }
147
148    /// Add a scenario result and update summary.
149    pub fn add_scenario(&mut self, result: ScenarioResult) {
150        match result.status {
151            ScenarioStatus::Pass => self.summary.passed += 1,
152            ScenarioStatus::Fail => self.summary.failed += 1,
153            ScenarioStatus::Skipped => self.summary.skipped += 1,
154            ScenarioStatus::Error => self.summary.errors += 1,
155        }
156        self.summary.total += 1;
157        self.scenarios.push(result);
158    }
159
160    /// Set total elapsed time.
161    pub fn set_elapsed(&mut self, elapsed: Duration) {
162        self.elapsed_ms = elapsed.as_millis() as u64;
163    }
164
165    /// Serialize to pretty JSON.
166    pub fn to_json(&self) -> String {
167        serde_json::to_string_pretty(self).expect("oracle report serialization should not fail")
168    }
169
170    /// Write report to a JSON file.
171    pub fn write_json(&self, path: &Path) -> std::io::Result<()> {
172        if let Some(parent) = path.parent() {
173            fs::create_dir_all(parent)?;
174        }
175        fs::write(path, self.to_json())
176    }
177
178    /// Generate a human-readable Markdown report.
179    pub fn to_markdown(&self) -> String {
180        let mut md = String::new();
181        md.push_str("# Oracle Differential Report\n\n");
182        md.push_str(&format!("**pproxy version:** {}\n", self.pproxy_version));
183        md.push_str(&format!(
184            "**eggress commit:** {}\n",
185            self.eggress_commit.as_deref().unwrap_or("unknown")
186        ));
187        md.push_str(&format!("**platform:** {}\n", self.os_platform));
188        md.push_str(&format!("**elapsed:** {}ms\n\n", self.elapsed_ms));
189
190        md.push_str("## Summary\n\n");
191        md.push_str("| Metric | Count |\n|--------|-------|\n");
192        md.push_str(&format!("| Total | {} |\n", self.summary.total));
193        md.push_str(&format!("| Passed | {} |\n", self.summary.passed));
194        md.push_str(&format!("| Failed | {} |\n", self.summary.failed));
195        md.push_str(&format!("| Skipped | {} |\n", self.summary.skipped));
196        md.push_str(&format!("| Errors | {} |\n\n", self.summary.errors));
197
198        let mut by_category: BTreeMap<String, Vec<&ScenarioResult>> = BTreeMap::new();
199        for s in &self.scenarios {
200            let cat = format!("{:?}", s.category);
201            by_category.entry(cat).or_default().push(s);
202        }
203
204        for (cat, scenarios) in &by_category {
205            md.push_str(&format!("## {}\n\n", cat));
206            for s in scenarios {
207                let status_icon = match s.status {
208                    ScenarioStatus::Pass => "✅",
209                    ScenarioStatus::Fail => "❌",
210                    ScenarioStatus::Skipped => "⏭️",
211                    ScenarioStatus::Error => "⚠️",
212                };
213                md.push_str(&format!(
214                    "### {} {} {}\n\n",
215                    status_icon, s.id, s.description
216                ));
217
218                if let Some(err) = &s.error {
219                    md.push_str(&format!("**Error:** {}\n\n", err));
220                }
221                if let Some(skip) = &s.skip_reason {
222                    md.push_str(&format!("**Skip reason:** {}\n\n", skip));
223                }
224
225                if !s.comparisons.is_empty() {
226                    md.push_str("| Dimension | Match | Pproxy | Eggress |\n");
227                    md.push_str("|-----------|-------|--------|----------|\n");
228                    for c in &s.comparisons {
229                        let match_icon = if c.matched { "✅" } else { "❌" };
230                        md.push_str(&format!(
231                            "| {} | {} | {} | {} |\n",
232                            c.dimension,
233                            match_icon,
234                            truncate_md(&c.pproxy_value, 50),
235                            truncate_md(&c.eggress_value, 50)
236                        ));
237                    }
238                    md.push('\n');
239                }
240
241                if !s.divergence_ids.is_empty() {
242                    md.push_str(&format!(
243                        "**Divergences:** {}\n\n",
244                        s.divergence_ids.join(", ")
245                    ));
246                }
247            }
248        }
249
250        md
251    }
252
253    /// Check manifest consistency: verify every scenario references valid capability IDs.
254    pub fn check_manifest_consistency(&self, valid_capability_ids: &[&str]) -> Vec<String> {
255        let mut warnings = Vec::new();
256        for s in &self.scenarios {
257            for cap_id in &s.capability_ids {
258                if !valid_capability_ids.contains(&cap_id.as_str()) {
259                    warnings.push(format!(
260                        "scenario '{}' references unknown capability '{}'",
261                        s.id, cap_id
262                    ));
263                }
264            }
265        }
266        warnings
267    }
268
269    /// Filter scenarios by certification profile.
270    pub fn scenarios_for_profile(&self, profile: CertificationProfile) -> Vec<&ScenarioResult> {
271        self.scenarios
272            .iter()
273            .filter(|s| s.certification_profile == Some(profile))
274            .collect()
275    }
276}
277
278impl ScenarioResult {
279    pub fn new(id: &str, category: ScenarioCategory, description: &str) -> Self {
280        Self {
281            id: id.to_string(),
282            category,
283            description: description.to_string(),
284            status: ScenarioStatus::Skipped,
285            comparisons: Vec::new(),
286            elapsed_ms: 0,
287            error: None,
288            skip_reason: None,
289            pproxy_observation: None,
290            eggress_observation: None,
291            timing_tolerance_ms: None,
292            divergence_ids: Vec::new(),
293            certification_profile: None,
294            capability_ids: Vec::new(),
295        }
296    }
297
298    pub fn with_status(mut self, status: ScenarioStatus) -> Self {
299        self.status = status;
300        self
301    }
302
303    pub fn with_elapsed(mut self, elapsed: Duration) -> Self {
304        self.elapsed_ms = elapsed.as_millis() as u64;
305        self
306    }
307
308    pub fn with_comparisons(mut self, comparisons: Vec<ComparisonResult>) -> Self {
309        self.comparisons = comparisons;
310        self
311    }
312
313    pub fn with_error(mut self, error: String) -> Self {
314        self.error = Some(error);
315        self
316    }
317
318    pub fn with_skip_reason(mut self, reason: String) -> Self {
319        self.skip_reason = Some(reason);
320        self.status = ScenarioStatus::Skipped;
321        self
322    }
323
324    pub fn with_pproxy_observation(mut self, obs: ProxyObservation) -> Self {
325        self.pproxy_observation = Some(obs);
326        self
327    }
328
329    pub fn with_eggress_observation(mut self, obs: ProxyObservation) -> Self {
330        self.eggress_observation = Some(obs);
331        self
332    }
333
334    pub fn with_timing_tolerance(mut self, tolerance_ms: u64) -> Self {
335        self.timing_tolerance_ms = Some(tolerance_ms);
336        self
337    }
338
339    pub fn with_divergences(mut self, ids: Vec<String>) -> Self {
340        self.divergence_ids = ids;
341        self
342    }
343
344    pub fn with_certification_profile(mut self, profile: CertificationProfile) -> Self {
345        self.certification_profile = Some(profile);
346        self
347    }
348
349    pub fn with_capability_ids(mut self, ids: Vec<String>) -> Self {
350        self.capability_ids = ids;
351        self
352    }
353}
354
355fn truncate_md(s: &str, max: usize) -> String {
356    if s.len() <= max {
357        s.replace('|', "\\|")
358    } else {
359        format!("{}... ({} bytes)", s[..max].replace('|', "\\|"), s.len())
360    }
361}
362
363/// Normalize a value for comparison (strip ports, versions, etc.).
364pub fn normalize_for_comparison(value: &str, scenario_id: &str) -> String {
365    let mut result = value.to_string();
366
367    if let Ok(re) = regex::Regex::new(r":\d{4,5}") {
368        result = re.replace_all(&result, ":PORT").to_string();
369    }
370
371    if scenario_id.starts_with("cli.") {
372        for prefix in &["INFO:", "WARNING:", "DEBUG:", "Listen: "] {
373            result = result.replace(prefix, "");
374        }
375    }
376
377    result = result.replace("pproxy/", "");
378    result = result.replace("eggress/", "");
379
380    result.trim().to_string()
381}
382
383/// Create a comparison result.
384pub fn make_comparison(
385    dimension: &str,
386    pproxy_value: &str,
387    eggress_value: &str,
388) -> ComparisonResult {
389    let matched = pproxy_value == eggress_value;
390    let details = if matched {
391        None
392    } else {
393        Some(format!(
394            "pproxy: {}, eggress: {}",
395            truncate(pproxy_value, 200),
396            truncate(eggress_value, 200)
397        ))
398    };
399    ComparisonResult {
400        dimension: dimension.to_string(),
401        pproxy_value: pproxy_value.to_string(),
402        eggress_value: eggress_value.to_string(),
403        matched,
404        details,
405    }
406}
407
408fn truncate(s: &str, max_len: usize) -> String {
409    if s.len() <= max_len {
410        s.to_string()
411    } else {
412        format!("{}... ({} bytes total)", &s[..max_len], s.len())
413    }
414}
415
416fn detect_pproxy_version() -> Option<String> {
417    let python = match std::panic::catch_unwind(crate::differential::find_python_binary) {
418        Ok(p) => p,
419        Err(_) => return None,
420    };
421    Command::new(&python)
422        .args([
423            "-c",
424            "import pproxy; print(getattr(pproxy, '__version__', 'unknown'))",
425        ])
426        .stdout(std::process::Stdio::piped())
427        .stderr(std::process::Stdio::null())
428        .output()
429        .ok()
430        .and_then(|o| {
431            if o.status.success() {
432                String::from_utf8(o.stdout)
433                    .ok()
434                    .map(|s| s.trim().to_string())
435            } else {
436                None
437            }
438        })
439}
440
441fn detect_eggress_commit() -> Option<String> {
442    if let Ok(val) = std::env::var("EGRESS_COMMIT") {
443        if !val.is_empty() {
444            return Some(val);
445        }
446    }
447    Command::new("git")
448        .args(["rev-parse", "--short", "HEAD"])
449        .output()
450        .ok()
451        .and_then(|o| {
452            if o.status.success() {
453                String::from_utf8(o.stdout)
454                    .ok()
455                    .map(|s| s.trim().to_string())
456            } else {
457                None
458            }
459        })
460}
461
462fn detect_python_version() -> Option<String> {
463    Command::new("python3")
464        .arg("--version")
465        .output()
466        .ok()
467        .and_then(|o| {
468            if o.status.success() {
469                String::from_utf8(o.stdout)
470                    .ok()
471                    .map(|s| s.trim().to_string())
472            } else {
473                None
474            }
475        })
476}
477
478fn detect_rust_version() -> String {
479    Command::new("rustc")
480        .arg("--version")
481        .output()
482        .ok()
483        .and_then(|o| {
484            if o.status.success() {
485                String::from_utf8(o.stdout)
486                    .ok()
487                    .map(|s| s.trim().to_string())
488            } else {
489                None
490            }
491        })
492        .unwrap_or_else(|| "unknown".to_string())
493}
494
495#[cfg(test)]
496mod tests {
497    use super::*;
498
499    #[test]
500    fn report_json_roundtrip() {
501        let mut report = OracleReport::new();
502        report.add_scenario(
503            ScenarioResult::new("test", ScenarioCategory::CliDefaults, "test scenario")
504                .with_status(ScenarioStatus::Pass)
505                .with_elapsed(Duration::from_millis(100)),
506        );
507        report.set_elapsed(Duration::from_secs(5));
508
509        let json = report.to_json();
510        let parsed: OracleReport = serde_json::from_str(&json).unwrap();
511        assert_eq!(parsed.scenarios.len(), 1);
512        assert_eq!(parsed.summary.total, 1);
513        assert_eq!(parsed.summary.passed, 1);
514        assert_eq!(parsed.elapsed_ms, 5000);
515    }
516
517    #[test]
518    fn make_comparison_match() {
519        let comp = make_comparison("payload", "hello", "hello");
520        assert!(comp.matched);
521        assert!(comp.details.is_none());
522    }
523
524    #[test]
525    fn make_comparison_mismatch() {
526        let comp = make_comparison("payload", "hello", "world");
527        assert!(!comp.matched);
528        assert!(comp.details.is_some());
529    }
530
531    #[test]
532    fn scenario_category_serde() {
533        let cat = ScenarioCategory::CliDefaults;
534        let json = serde_json::to_string(&cat).unwrap();
535        assert_eq!(json, "\"cli_defaults\"");
536    }
537
538    #[test]
539    fn markdown_report_generation() {
540        let mut report = OracleReport::new();
541        report.add_scenario(
542            ScenarioResult::new("test.pass", ScenarioCategory::CliDefaults, "passing test")
543                .with_status(ScenarioStatus::Pass)
544                .with_comparisons(vec![make_comparison("payload", "hello", "hello")]),
545        );
546        report.add_scenario(
547            ScenarioResult::new("test.fail", ScenarioCategory::HttpSocksTcp, "failing test")
548                .with_status(ScenarioStatus::Fail)
549                .with_error("mismatch".to_string()),
550        );
551
552        let md = report.to_markdown();
553        assert!(md.contains("# Oracle Differential Report"));
554        assert!(md.contains("test.pass"));
555        assert!(md.contains("test.fail"));
556        assert!(md.contains("✅"));
557        assert!(md.contains("❌"));
558    }
559
560    #[test]
561    fn manifest_consistency_check() {
562        let mut report = OracleReport::new();
563        report.add_scenario(
564            ScenarioResult::new("test", ScenarioCategory::CliDefaults, "test")
565                .with_capability_ids(vec!["valid.cap".to_string(), "unknown.cap".to_string()]),
566        );
567
568        let warnings = report.check_manifest_consistency(&["valid.cap"]);
569        assert_eq!(warnings.len(), 1);
570        assert!(warnings[0].contains("unknown.cap"));
571    }
572
573    #[test]
574    fn manifest_consistency_no_warnings_for_valid() {
575        let mut report = OracleReport::new();
576        report.add_scenario(
577            ScenarioResult::new("test", ScenarioCategory::CliDefaults, "test")
578                .with_capability_ids(vec!["cap.a".to_string(), "cap.b".to_string()]),
579        );
580
581        let warnings = report.check_manifest_consistency(&["cap.a", "cap.b"]);
582        assert!(warnings.is_empty());
583    }
584
585    #[test]
586    fn tier_filtering() {
587        let mut report = OracleReport::new();
588        report.add_scenario(
589            ScenarioResult::new("fast", ScenarioCategory::CliDefaults, "fast")
590                .with_certification_profile(CertificationProfile::Differential),
591        );
592        report.add_scenario(
593            ScenarioResult::new("core", ScenarioCategory::HttpSocksTcp, "core")
594                .with_certification_profile(CertificationProfile::Differential),
595        );
596        report.add_scenario(ScenarioResult::new(
597            "no_profile",
598            ScenarioCategory::Chains,
599            "no profile",
600        ));
601
602        let differential = report.scenarios_for_profile(CertificationProfile::Differential);
603        assert_eq!(differential.len(), 2);
604
605        let platform = report.scenarios_for_profile(CertificationProfile::Platform);
606        assert!(platform.is_empty());
607    }
608
609    #[test]
610    fn scenario_result_builder() {
611        let result = ScenarioResult::new("id", ScenarioCategory::Chains, "desc")
612            .with_status(ScenarioStatus::Pass)
613            .with_elapsed(Duration::from_secs(1))
614            .with_comparisons(vec![make_comparison("dim", "a", "a")])
615            .with_divergences(vec!["div1".to_string()])
616            .with_certification_profile(CertificationProfile::Differential)
617            .with_capability_ids(vec!["cap1".to_string()]);
618
619        assert_eq!(result.status, ScenarioStatus::Pass);
620        assert_eq!(result.elapsed_ms, 1000);
621        assert_eq!(result.divergence_ids, vec!["div1"]);
622        assert_eq!(
623            result.certification_profile,
624            Some(CertificationProfile::Differential)
625        );
626        assert_eq!(result.capability_ids, vec!["cap1"]);
627    }
628
629    #[test]
630    fn scenario_result_builder_skip_reason_sets_status() {
631        let result = ScenarioResult::new("id", ScenarioCategory::CliDefaults, "desc")
632            .with_status(ScenarioStatus::Pass)
633            .with_skip_reason("missing deps".to_string());
634        assert_eq!(result.status, ScenarioStatus::Skipped);
635    }
636
637    #[test]
638    fn certification_profile_serde() {
639        let profile = CertificationProfile::Differential;
640        let json = serde_json::to_string(&profile).unwrap();
641        assert_eq!(json, "\"differential\"");
642        let parsed: CertificationProfile = serde_json::from_str(&json).unwrap();
643        assert_eq!(parsed, profile);
644    }
645
646    #[test]
647    fn markdown_includes_divergences() {
648        let mut report = OracleReport::new();
649        report.add_scenario(
650            ScenarioResult::new("div_test", ScenarioCategory::Udp, "udp test")
651                .with_status(ScenarioStatus::Pass)
652                .with_divergences(vec!["div.timing".to_string(), "div.payload".to_string()]),
653        );
654
655        let md = report.to_markdown();
656        assert!(md.contains("**Divergences:** div.timing, div.payload"));
657    }
658
659    #[test]
660    fn markdown_includes_skip_reason() {
661        let mut report = OracleReport::new();
662        report.add_scenario(
663            ScenarioResult::new("skip_test", ScenarioCategory::CliDefaults, "skipped")
664                .with_skip_reason("needs root".to_string()),
665        );
666
667        let md = report.to_markdown();
668        assert!(md.contains("**Skip reason:** needs root"));
669        assert!(md.contains("⏭️"));
670    }
671
672    #[test]
673    fn markdown_truncates_long_values() {
674        let mut report = OracleReport::new();
675        let long_value = "x".repeat(200);
676        report.add_scenario(
677            ScenarioResult::new("long", ScenarioCategory::HttpSocksTcp, "long values")
678                .with_status(ScenarioStatus::Fail)
679                .with_comparisons(vec![make_comparison("dim", &long_value, "short")]),
680        );
681
682        let md = report.to_markdown();
683        assert!(md.contains("... (200 bytes)"));
684    }
685
686    #[test]
687    fn json_roundtrip_preserves_new_fields() {
688        let mut report = OracleReport::new();
689        report.add_scenario(
690            ScenarioResult::new("full", ScenarioCategory::Chains, "full fields")
691                .with_status(ScenarioStatus::Pass)
692                .with_timing_tolerance(50)
693                .with_divergences(vec!["d1".to_string()])
694                .with_certification_profile(CertificationProfile::Differential)
695                .with_capability_ids(vec!["cap.x".to_string()]),
696        );
697
698        let json = report.to_json();
699        let parsed: OracleReport = serde_json::from_str(&json).unwrap();
700        let s = &parsed.scenarios[0];
701        assert_eq!(s.timing_tolerance_ms, Some(50));
702        assert_eq!(s.divergence_ids, vec!["d1"]);
703        assert_eq!(
704            s.certification_profile,
705            Some(CertificationProfile::Differential)
706        );
707        assert_eq!(s.capability_ids, vec!["cap.x"]);
708    }
709}
eggress_testkit/oracle/report.rs

eggress_testkit/oracle/
report.rs
