batuta/oracle/falsify/
report.rs

1//! Falsification Report Generation
2//!
3//! Generates reports from test execution results.
4
5use serde::{Deserialize, Serialize};
6use std::fmt::Write as FmtWrite;
7
8/// Test outcome
9#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
10pub enum TestOutcome {
11    /// Test passed (claim not falsified)
12    Passed,
13    /// Test failed (claim falsified - this is good!)
14    Falsified,
15    /// Test skipped
16    Skipped,
17    /// Test errored (infrastructure issue)
18    Error,
19}
20
21/// Falsification report
22#[derive(Debug, Clone, Serialize, Deserialize)]
23pub struct FalsificationReport {
24    /// Spec name
25    pub spec_name: String,
26    /// Test results
27    pub results: Vec<TestResult>,
28    /// Summary statistics
29    pub summary: FalsificationSummary,
30    /// Generation timestamp
31    pub generated_at: String,
32}
33
34/// Single test result
35#[derive(Debug, Clone, Serialize, Deserialize)]
36pub struct TestResult {
37    /// Test ID
38    pub id: String,
39    /// Test name
40    pub name: String,
41    /// Category
42    pub category: String,
43    /// Points
44    pub points: u32,
45    /// Outcome
46    pub outcome: TestOutcome,
47    /// Error message if any
48    pub error: Option<String>,
49    /// Evidence collected
50    pub evidence: Vec<String>,
51}
52
53/// Summary statistics
54#[derive(Debug, Clone, Default, Serialize, Deserialize)]
55pub struct FalsificationSummary {
56    /// Total tests
57    pub total_tests: usize,
58    /// Total points
59    pub total_points: u32,
60    /// Tests passed
61    pub passed: usize,
62    /// Tests falsified (failures = success in Popperian methodology!)
63    pub falsified: usize,
64    /// Tests skipped
65    pub skipped: usize,
66    /// Tests errored
67    pub errors: usize,
68    /// Falsification rate (target: 5-15%)
69    pub falsification_rate: f64,
70    /// Points by category
71    pub points_by_category: std::collections::HashMap<String, CategoryStats>,
72}
73
74/// Category statistics
75#[derive(Debug, Clone, Default, Serialize, Deserialize)]
76pub struct CategoryStats {
77    pub total: u32,
78    pub passed: u32,
79    pub falsified: u32,
80}
81
82impl FalsificationReport {
83    /// Create a new report
84    pub fn new(spec_name: String) -> Self {
85        Self {
86            spec_name,
87            results: Vec::new(),
88            summary: FalsificationSummary::default(),
89            generated_at: chrono::Utc::now().to_rfc3339(),
90        }
91    }
92
93    /// Add a test result
94    pub fn add_result(&mut self, result: TestResult) {
95        self.results.push(result);
96    }
97
98    /// Finalize and compute summary
99    pub fn finalize(&mut self) {
100        self.summary.total_tests = self.results.len();
101        self.summary.total_points = self.results.iter().map(|r| r.points).sum();
102
103        for result in &self.results {
104            match result.outcome {
105                TestOutcome::Passed => self.summary.passed += 1,
106                TestOutcome::Falsified => self.summary.falsified += 1,
107                TestOutcome::Skipped => self.summary.skipped += 1,
108                TestOutcome::Error => self.summary.errors += 1,
109            }
110
111            let entry = self.summary.points_by_category.entry(result.category.clone()).or_default();
112            entry.total += result.points;
113            match result.outcome {
114                TestOutcome::Passed => entry.passed += result.points,
115                TestOutcome::Falsified => entry.falsified += result.points,
116                _ => {}
117            }
118        }
119
120        let executed = self.summary.passed + self.summary.falsified;
121        if executed > 0 {
122            self.summary.falsification_rate = (self.summary.falsified as f64) / (executed as f64);
123        }
124    }
125
126    /// Format as markdown
127    pub fn format_markdown(&self) -> String {
128        let mut out = String::new();
129
130        writeln!(out, "# Falsification Report: {}", self.spec_name).ok();
131        writeln!(out).ok();
132        writeln!(out, "**Generated**: {}", self.generated_at).ok();
133        writeln!(out, "**Total Points**: {}", self.summary.total_points).ok();
134        writeln!(out, "**Falsifications Found**: {} (target: 5-15%)", self.summary.falsified).ok();
135        writeln!(out).ok();
136
137        writeln!(out, "## Summary").ok();
138        writeln!(out).ok();
139        writeln!(out, "| Category | Points | Passed | Failed | Pass Rate |").ok();
140        writeln!(out, "|----------|--------|--------|--------|-----------|").ok();
141
142        for (category, stats) in &self.summary.points_by_category {
143            let pass_rate = if stats.total > 0 {
144                (stats.passed as f64 / stats.total as f64) * 100.0
145            } else {
146                0.0
147            };
148            writeln!(
149                out,
150                "| {} | {} | {} | {} | {:.0}% |",
151                category, stats.total, stats.passed, stats.falsified, pass_rate
152            )
153            .ok();
154        }
155        writeln!(out).ok();
156
157        // Verdict
158        let verdict =
159            if self.summary.falsification_rate >= 0.05 && self.summary.falsification_rate <= 0.15 {
160                "Healthy falsification rate - specification is well-tested"
161            } else if self.summary.falsification_rate < 0.05 {
162                "Low falsification rate - consider more edge cases"
163            } else {
164                "High falsification rate - specification needs hardening"
165            };
166
167        writeln!(
168            out,
169            "**Verdict**: {:.1}% falsification rate - {}",
170            self.summary.falsification_rate * 100.0,
171            verdict
172        )
173        .ok();
174        writeln!(out).ok();
175
176        // Falsifications (failures = success!)
177        if self.summary.falsified > 0 {
178            writeln!(out, "## Falsifications (Failures = Success!)").ok();
179            writeln!(out).ok();
180
181            for result in &self.results {
182                if result.outcome == TestOutcome::Falsified {
183                    writeln!(out, "### {}: {}", result.id, result.name).ok();
184                    writeln!(out, "**Status**: FALSIFIED").ok();
185                    writeln!(out, "**Points**: {}", result.points).ok();
186                    if let Some(err) = &result.error {
187                        writeln!(out, "**Details**: {}", err).ok();
188                    }
189                    for evidence in &result.evidence {
190                        writeln!(out, "- {}", evidence).ok();
191                    }
192                    writeln!(out).ok();
193                }
194            }
195        }
196
197        out
198    }
199
200    /// Format as JSON
201    pub fn format_json(&self) -> String {
202        serde_json::to_string_pretty(self).unwrap_or_else(|_| "{}".to_string())
203    }
204
205    /// Format as text
206    pub fn format_text(&self) -> String {
207        let mut out = String::new();
208
209        writeln!(out, "FALSIFICATION REPORT: {}", self.spec_name).ok();
210        writeln!(out, "{}", "=".repeat(60)).ok();
211        writeln!(out).ok();
212
213        writeln!(out, "Total Points: {}", self.summary.total_points).ok();
214        writeln!(
215            out,
216            "Falsifications: {} ({:.1}%)",
217            self.summary.falsified,
218            self.summary.falsification_rate * 100.0
219        )
220        .ok();
221        writeln!(out).ok();
222
223        for result in &self.results {
224            let status = match result.outcome {
225                TestOutcome::Passed => "PASS",
226                TestOutcome::Falsified => "FAIL",
227                TestOutcome::Skipped => "SKIP",
228                TestOutcome::Error => "ERR",
229            };
230            writeln!(out, "[{}] {}: {} ({} pts)", status, result.id, result.name, result.points)
231                .ok();
232        }
233
234        out
235    }
236}
237
238#[cfg(test)]
239mod tests {
240    use super::*;
241
242    #[test]
243    fn test_report_creation() {
244        let report = FalsificationReport::new("test-spec".to_string());
245        assert_eq!(report.spec_name, "test-spec");
246    }
247
248    #[test]
249    fn test_report_finalize() {
250        let mut report = FalsificationReport::new("test".to_string());
251
252        report.add_result(TestResult {
253            id: "BC-001".to_string(),
254            name: "Test 1".to_string(),
255            category: "boundary".to_string(),
256            points: 5,
257            outcome: TestOutcome::Passed,
258            error: None,
259            evidence: vec![],
260        });
261
262        report.add_result(TestResult {
263            id: "BC-002".to_string(),
264            name: "Test 2".to_string(),
265            category: "boundary".to_string(),
266            points: 5,
267            outcome: TestOutcome::Falsified,
268            error: Some("Found edge case".to_string()),
269            evidence: vec!["Input: empty".to_string()],
270        });
271
272        report.finalize();
273
274        assert_eq!(report.summary.total_tests, 2);
275        assert_eq!(report.summary.passed, 1);
276        assert_eq!(report.summary.falsified, 1);
277        assert!((report.summary.falsification_rate - 0.5).abs() < 0.01);
278    }
279
280    #[test]
281    fn test_format_markdown() {
282        let mut report = FalsificationReport::new("test".to_string());
283        report.add_result(TestResult {
284            id: "BC-001".to_string(),
285            name: "Empty input".to_string(),
286            category: "boundary".to_string(),
287            points: 5,
288            outcome: TestOutcome::Passed,
289            error: None,
290            evidence: vec![],
291        });
292        report.finalize();
293
294        let md = report.format_markdown();
295        assert!(md.contains("Falsification Report"));
296        assert!(md.contains("boundary"));
297    }
298
299    #[test]
300    fn test_format_json() {
301        let mut report = FalsificationReport::new("json-test".to_string());
302        report.add_result(TestResult {
303            id: "BC-001".to_string(),
304            name: "Test".to_string(),
305            category: "boundary".to_string(),
306            points: 5,
307            outcome: TestOutcome::Passed,
308            error: None,
309            evidence: vec![],
310        });
311        report.finalize();
312
313        let json = report.format_json();
314        assert!(json.contains("json-test"));
315        assert!(json.contains("BC-001"));
316        assert!(json.contains("boundary"));
317    }
318
319    #[test]
320    fn test_format_text() {
321        let mut report = FalsificationReport::new("text-test".to_string());
322        report.add_result(TestResult {
323            id: "BC-001".to_string(),
324            name: "Test".to_string(),
325            category: "boundary".to_string(),
326            points: 5,
327            outcome: TestOutcome::Passed,
328            error: None,
329            evidence: vec![],
330        });
331        report.add_result(TestResult {
332            id: "BC-002".to_string(),
333            name: "Test 2".to_string(),
334            category: "boundary".to_string(),
335            points: 5,
336            outcome: TestOutcome::Falsified,
337            error: None,
338            evidence: vec![],
339        });
340        report.finalize();
341
342        let text = report.format_text();
343        assert!(text.contains("FALSIFICATION REPORT"));
344        assert!(text.contains("text-test"));
345        assert!(text.contains("[PASS]"));
346        assert!(text.contains("[FAIL]"));
347    }
348
349    #[test]
350    fn test_test_outcome_equality() {
351        assert_eq!(TestOutcome::Passed, TestOutcome::Passed);
352        assert_eq!(TestOutcome::Falsified, TestOutcome::Falsified);
353        assert_eq!(TestOutcome::Skipped, TestOutcome::Skipped);
354        assert_eq!(TestOutcome::Error, TestOutcome::Error);
355        assert_ne!(TestOutcome::Passed, TestOutcome::Falsified);
356    }
357
358    #[test]
359    fn test_falsification_summary_default() {
360        let summary = FalsificationSummary::default();
361        assert_eq!(summary.total_tests, 0);
362        assert_eq!(summary.total_points, 0);
363        assert_eq!(summary.passed, 0);
364        assert_eq!(summary.falsified, 0);
365        assert_eq!(summary.skipped, 0);
366        assert_eq!(summary.errors, 0);
367        assert!((summary.falsification_rate - 0.0).abs() < f64::EPSILON);
368    }
369
370    #[test]
371    fn test_category_stats_default() {
372        let stats = CategoryStats::default();
373        assert_eq!(stats.total, 0);
374        assert_eq!(stats.passed, 0);
375        assert_eq!(stats.falsified, 0);
376    }
377
378    #[test]
379    fn test_report_with_all_outcomes() {
380        let mut report = FalsificationReport::new("all-outcomes".to_string());
381
382        report.add_result(TestResult {
383            id: "T-001".to_string(),
384            name: "Passed".to_string(),
385            category: "test".to_string(),
386            points: 1,
387            outcome: TestOutcome::Passed,
388            error: None,
389            evidence: vec![],
390        });
391        report.add_result(TestResult {
392            id: "T-002".to_string(),
393            name: "Falsified".to_string(),
394            category: "test".to_string(),
395            points: 2,
396            outcome: TestOutcome::Falsified,
397            error: None,
398            evidence: vec![],
399        });
400        report.add_result(TestResult {
401            id: "T-003".to_string(),
402            name: "Skipped".to_string(),
403            category: "test".to_string(),
404            points: 3,
405            outcome: TestOutcome::Skipped,
406            error: None,
407            evidence: vec![],
408        });
409        report.add_result(TestResult {
410            id: "T-004".to_string(),
411            name: "Error".to_string(),
412            category: "test".to_string(),
413            points: 4,
414            outcome: TestOutcome::Error,
415            error: Some("Infra issue".to_string()),
416            evidence: vec![],
417        });
418
419        report.finalize();
420
421        assert_eq!(report.summary.total_tests, 4);
422        assert_eq!(report.summary.passed, 1);
423        assert_eq!(report.summary.falsified, 1);
424        assert_eq!(report.summary.skipped, 1);
425        assert_eq!(report.summary.errors, 1);
426    }
427
428    #[test]
429    fn test_report_format_text_status_codes() {
430        let mut report = FalsificationReport::new("status".to_string());
431        report.add_result(TestResult {
432            id: "T-001".to_string(),
433            name: "Skip".to_string(),
434            category: "test".to_string(),
435            points: 1,
436            outcome: TestOutcome::Skipped,
437            error: None,
438            evidence: vec![],
439        });
440        report.add_result(TestResult {
441            id: "T-002".to_string(),
442            name: "Err".to_string(),
443            category: "test".to_string(),
444            points: 1,
445            outcome: TestOutcome::Error,
446            error: None,
447            evidence: vec![],
448        });
449        report.finalize();
450
451        let text = report.format_text();
452        assert!(text.contains("[SKIP]"));
453        assert!(text.contains("[ERR]"));
454    }
455
456    #[test]
457    fn test_markdown_with_falsifications() {
458        let mut report = FalsificationReport::new("falsify-test".to_string());
459        report.add_result(TestResult {
460            id: "BC-001".to_string(),
461            name: "Edge case".to_string(),
462            category: "boundary".to_string(),
463            points: 5,
464            outcome: TestOutcome::Falsified,
465            error: Some("Assertion failed".to_string()),
466            evidence: vec!["Input: empty".to_string(), "Expected: error".to_string()],
467        });
468        report.finalize();
469
470        let md = report.format_markdown();
471        assert!(md.contains("Falsifications (Failures = Success!)"));
472        assert!(md.contains("BC-001"));
473        assert!(md.contains("FALSIFIED"));
474        assert!(md.contains("Assertion failed"));
475        assert!(md.contains("Input: empty"));
476    }
477
478    #[test]
479    fn test_markdown_healthy_rate() {
480        let mut report = FalsificationReport::new("healthy".to_string());
481        for i in 0..19 {
482            report.add_result(TestResult {
483                id: format!("T-{:03}", i),
484                name: format!("Test {}", i),
485                category: "test".to_string(),
486                points: 1,
487                outcome: TestOutcome::Passed,
488                error: None,
489                evidence: vec![],
490            });
491        }
492        report.add_result(TestResult {
493            id: "T-019".to_string(),
494            name: "Falsified".to_string(),
495            category: "test".to_string(),
496            points: 1,
497            outcome: TestOutcome::Falsified,
498            error: None,
499            evidence: vec![],
500        });
501        report.finalize();
502
503        // 1/20 = 5% = healthy
504        let md = report.format_markdown();
505        assert!(md.contains("well-tested"));
506    }
507
508    #[test]
509    fn test_markdown_low_falsification_rate() {
510        let mut report = FalsificationReport::new("low".to_string());
511        for i in 0..100 {
512            report.add_result(TestResult {
513                id: format!("T-{:03}", i),
514                name: format!("Test {}", i),
515                category: "test".to_string(),
516                points: 1,
517                outcome: TestOutcome::Passed,
518                error: None,
519                evidence: vec![],
520            });
521        }
522        report.finalize();
523
524        let md = report.format_markdown();
525        assert!(md.contains("more edge cases"));
526    }
527}
batuta/oracle/falsify/report.rs

batuta/oracle/falsify/
report.rs
