batuta/oracle/falsify/
mod.rs

1//! Popperian Falsification QA Generator
2//!
3//! Generates 100-point falsification test suites from specifications.
4//! Following Popper's principle: "The wrong view of science betrays itself
5//! in the craving to be right."
6//!
7//! # Categories (100 points total)
8//!
9//! - Boundary Conditions (20 points): BC-001..005
10//! - Invariant Violations (20 points): INV-001..004
11//! - Numerical Stability (20 points): NUM-001..003
12//! - Concurrency (15 points): CONC-001..003
13//! - Resource Exhaustion (15 points): RES-001..003
14//! - Cross-Implementation Parity (10 points): PAR-001..002
15//!
16//! # Toyota Production System Principles
17//!
18//! - **Jidoka**: Stop-on-error in test execution
19//! - **Genchi Genbutsu**: Evidence-based verification
20//! - **Kaizen**: Continuous improvement via falsification rate tracking
21
22pub mod categories;
23pub mod generator;
24pub mod parser;
25pub mod report;
26pub mod template;
27
28pub use generator::{FalsifyGenerator, GeneratedTest, TargetLanguage};
29pub use parser::SpecParser;
30#[allow(unused_imports)]
31pub use parser::{ParsedRequirement, ParsedSpec};
32#[allow(unused_imports)]
33pub use report::{FalsificationReport, FalsificationSummary, TestOutcome};
34pub use template::FalsificationTemplate;
35#[allow(unused_imports)]
36pub use template::{CategoryTemplate, TestTemplate};
37
38/// Main Falsification Engine
39///
40/// Coordinates test generation from specifications.
41#[derive(Debug)]
42pub struct FalsifyEngine {
43    /// 100-point template
44    template: FalsificationTemplate,
45    /// Specification parser
46    parser: SpecParser,
47    /// Test generator
48    generator: FalsifyGenerator,
49}
50
51impl FalsifyEngine {
52    /// Create a new falsification engine with default template
53    pub fn new() -> Self {
54        Self {
55            template: FalsificationTemplate::default(),
56            parser: SpecParser::new(),
57            generator: FalsifyGenerator::new(),
58        }
59    }
60
61    /// Generate falsification suite from a specification file
62    pub fn generate_from_spec(
63        &self,
64        spec_path: &std::path::Path,
65        language: TargetLanguage,
66    ) -> anyhow::Result<GeneratedSuite> {
67        // Parse spec
68        let spec = self.parser.parse_file(spec_path)?;
69
70        // Generate tests
71        let tests = self.generator.generate(&spec, &self.template, language)?;
72
73        Ok(GeneratedSuite {
74            spec_name: spec.name.clone(),
75            language,
76            tests,
77            total_points: self.template.total_points(),
78        })
79    }
80
81    /// Generate with custom point allocation
82    pub fn generate_with_points(
83        &self,
84        spec_path: &std::path::Path,
85        language: TargetLanguage,
86        target_points: u32,
87    ) -> anyhow::Result<GeneratedSuite> {
88        let spec = self.parser.parse_file(spec_path)?;
89        let template = self.template.scale_to_points(target_points);
90        let tests = self.generator.generate(&spec, &template, language)?;
91
92        Ok(GeneratedSuite {
93            spec_name: spec.name.clone(),
94            language,
95            tests,
96            total_points: target_points,
97        })
98    }
99
100    /// Get the template for inspection
101    pub fn template(&self) -> &FalsificationTemplate {
102        &self.template
103    }
104}
105
106impl Default for FalsifyEngine {
107    fn default() -> Self {
108        Self::new()
109    }
110}
111
112/// A generated test suite
113#[derive(Debug)]
114pub struct GeneratedSuite {
115    /// Name from specification
116    pub spec_name: String,
117    /// Target language
118    pub language: TargetLanguage,
119    /// Generated tests
120    pub tests: Vec<GeneratedTest>,
121    /// Total points
122    pub total_points: u32,
123}
124
125impl GeneratedSuite {
126    /// Format as code string
127    pub fn to_code(&self) -> String {
128        self.generator_code()
129    }
130
131    /// Generate code for target language
132    fn generator_code(&self) -> String {
133        match self.language {
134            TargetLanguage::Rust => self.to_rust(),
135            TargetLanguage::Python => self.to_python(),
136        }
137    }
138
139    /// Generate Rust test file
140    fn to_rust(&self) -> String {
141        let mut out = String::new();
142        out.push_str(&format!("//! Falsification Suite: {}\n", self.spec_name));
143        out.push_str(&format!("//! Total Points: {}\n", self.total_points));
144        out.push_str("//! Generated by batuta oracle falsify\n\n");
145        out.push_str("#![cfg(test)]\n\n");
146        out.push_str("use proptest::prelude::*;\n\n");
147
148        // Group by category
149        let mut current_category = String::new();
150        for test in &self.tests {
151            if test.category != current_category {
152                current_category = test.category.clone();
153                out.push_str(&format!(
154                    "\n// {:=<60}\n",
155                    format!(" {} ", current_category.to_uppercase())
156                ));
157            }
158
159            out.push_str(&format!("\n/// {}: {}\n", test.id, test.name));
160            out.push_str(&format!("/// Points: {}\n", test.points));
161            out.push_str(&test.code);
162            out.push('\n');
163        }
164
165        out
166    }
167
168    /// Generate Python test file
169    fn to_python(&self) -> String {
170        let mut out = String::new();
171        out.push_str(&format!("\"\"\"Falsification Suite: {}\n\n", self.spec_name));
172        out.push_str(&format!("Total Points: {}\n", self.total_points));
173        out.push_str("Generated by batuta oracle falsify\n\"\"\"\n\n");
174        out.push_str("import pytest\n");
175        out.push_str("from hypothesis import given, strategies as st\n\n");
176
177        // Group by category
178        let mut current_category = String::new();
179        for test in &self.tests {
180            if test.category != current_category {
181                current_category = test.category.clone();
182                out.push_str(&format!(
183                    "\n# {:=<60}\n",
184                    format!(" {} ", current_category.to_uppercase())
185                ));
186            }
187
188            out.push_str(&format!("\ndef test_{}():\n", test.id.to_lowercase().replace('-', "_")));
189            out.push_str(&format!("    \"\"\"{}: {}\n", test.id, test.name));
190            out.push_str(&format!("    Points: {}\n    \"\"\"\n", test.points));
191            out.push_str(&test.code);
192            out.push('\n');
193        }
194
195        out
196    }
197
198    /// Get tests by category
199    pub fn tests_by_category(&self) -> std::collections::HashMap<String, Vec<&GeneratedTest>> {
200        let mut map = std::collections::HashMap::new();
201        for test in &self.tests {
202            map.entry(test.category.clone()).or_insert_with(Vec::new).push(test);
203        }
204        map
205    }
206
207    /// Get summary statistics
208    pub fn summary(&self) -> SuiteSummary {
209        let mut points_by_category = std::collections::HashMap::new();
210        for test in &self.tests {
211            *points_by_category.entry(test.category.clone()).or_insert(0u32) += test.points;
212        }
213
214        SuiteSummary {
215            spec_name: self.spec_name.clone(),
216            total_tests: self.tests.len(),
217            total_points: self.total_points,
218            points_by_category,
219        }
220    }
221}
222
223/// Summary of a generated suite
224#[derive(Debug)]
225pub struct SuiteSummary {
226    pub spec_name: String,
227    pub total_tests: usize,
228    pub total_points: u32,
229    pub points_by_category: std::collections::HashMap<String, u32>,
230}
231
232#[cfg(test)]
233mod tests {
234    use super::*;
235    use template::TestSeverity;
236
237    #[test]
238    fn test_falsify_engine_creation() {
239        let engine = FalsifyEngine::new();
240        assert_eq!(engine.template.total_points(), 100);
241    }
242
243    #[test]
244    fn test_template_categories() {
245        let engine = FalsifyEngine::new();
246        let template = engine.template();
247        assert!(!template.categories.is_empty());
248    }
249
250    #[test]
251    fn test_falsify_engine_default() {
252        let engine = FalsifyEngine::default();
253        assert_eq!(engine.template.total_points(), 100);
254    }
255
256    #[test]
257    fn test_generated_suite_to_code_rust() {
258        let suite = GeneratedSuite {
259            spec_name: "test-spec".to_string(),
260            language: TargetLanguage::Rust,
261            tests: vec![GeneratedTest {
262                id: "BC-001".to_string(),
263                name: "Boundary test".to_string(),
264                category: "boundary".to_string(),
265                points: 4,
266                severity: TestSeverity::High,
267                code: "#[test]\nfn test_boundary() {}".to_string(),
268            }],
269            total_points: 100,
270        };
271        let code = suite.to_code();
272        assert!(code.contains("test-spec"));
273        assert!(code.contains("BC-001"));
274        assert!(code.contains("proptest"));
275    }
276
277    #[test]
278    fn test_generated_suite_to_code_python() {
279        let suite = GeneratedSuite {
280            spec_name: "test-spec".to_string(),
281            language: TargetLanguage::Python,
282            tests: vec![GeneratedTest {
283                id: "BC-001".to_string(),
284                name: "Boundary test".to_string(),
285                category: "boundary".to_string(),
286                points: 4,
287                severity: TestSeverity::High,
288                code: "    pass".to_string(),
289            }],
290            total_points: 100,
291        };
292        let code = suite.to_code();
293        assert!(code.contains("test-spec"));
294        assert!(code.contains("pytest"));
295        assert!(code.contains("hypothesis"));
296    }
297
298    #[test]
299    fn test_generated_suite_tests_by_category() {
300        let suite = GeneratedSuite {
301            spec_name: "test".to_string(),
302            language: TargetLanguage::Rust,
303            tests: vec![
304                GeneratedTest {
305                    id: "BC-001".to_string(),
306                    name: "Test 1".to_string(),
307                    category: "boundary".to_string(),
308                    points: 4,
309                    severity: TestSeverity::High,
310                    code: String::new(),
311                },
312                GeneratedTest {
313                    id: "INV-001".to_string(),
314                    name: "Test 2".to_string(),
315                    category: "invariant".to_string(),
316                    points: 5,
317                    severity: TestSeverity::Critical,
318                    code: String::new(),
319                },
320                GeneratedTest {
321                    id: "BC-002".to_string(),
322                    name: "Test 3".to_string(),
323                    category: "boundary".to_string(),
324                    points: 4,
325                    severity: TestSeverity::Medium,
326                    code: String::new(),
327                },
328            ],
329            total_points: 13,
330        };
331        let by_cat = suite.tests_by_category();
332        assert_eq!(by_cat.len(), 2);
333        assert_eq!(by_cat.get("boundary").expect("key not found").len(), 2);
334        assert_eq!(by_cat.get("invariant").expect("key not found").len(), 1);
335    }
336
337    #[test]
338    fn test_generated_suite_summary() {
339        let suite = GeneratedSuite {
340            spec_name: "my-spec".to_string(),
341            language: TargetLanguage::Rust,
342            tests: vec![
343                GeneratedTest {
344                    id: "BC-001".to_string(),
345                    name: "Test 1".to_string(),
346                    category: "boundary".to_string(),
347                    points: 4,
348                    severity: TestSeverity::High,
349                    code: String::new(),
350                },
351                GeneratedTest {
352                    id: "BC-002".to_string(),
353                    name: "Test 2".to_string(),
354                    category: "boundary".to_string(),
355                    points: 4,
356                    severity: TestSeverity::High,
357                    code: String::new(),
358                },
359            ],
360            total_points: 8,
361        };
362        let summary = suite.summary();
363        assert_eq!(summary.spec_name, "my-spec");
364        assert_eq!(summary.total_tests, 2);
365        assert_eq!(summary.total_points, 8);
366        assert_eq!(*summary.points_by_category.get("boundary").expect("key not found"), 8);
367    }
368
369    #[test]
370    fn test_suite_summary_fields() {
371        let summary = SuiteSummary {
372            spec_name: "test".to_string(),
373            total_tests: 10,
374            total_points: 100,
375            points_by_category: std::collections::HashMap::new(),
376        };
377        assert_eq!(summary.spec_name, "test");
378        assert_eq!(summary.total_tests, 10);
379        assert_eq!(summary.total_points, 100);
380    }
381
382    #[test]
383    fn test_generated_suite_rust_code_format() {
384        let suite = GeneratedSuite {
385            spec_name: "spec".to_string(),
386            language: TargetLanguage::Rust,
387            tests: vec![
388                GeneratedTest {
389                    id: "BC-001".to_string(),
390                    name: "First".to_string(),
391                    category: "boundary".to_string(),
392                    points: 4,
393                    severity: TestSeverity::High,
394                    code: "// code".to_string(),
395                },
396                GeneratedTest {
397                    id: "INV-001".to_string(),
398                    name: "Second".to_string(),
399                    category: "invariant".to_string(),
400                    points: 5,
401                    severity: TestSeverity::Critical,
402                    code: "// more".to_string(),
403                },
404            ],
405            total_points: 9,
406        };
407        let code = suite.to_code();
408        // Should have category headers
409        assert!(code.contains("BOUNDARY"));
410        assert!(code.contains("INVARIANT"));
411        // Should have cfg test
412        assert!(code.contains("#![cfg(test)]"));
413    }
414
415    #[test]
416    fn test_generated_suite_python_code_format() {
417        let suite = GeneratedSuite {
418            spec_name: "spec".to_string(),
419            language: TargetLanguage::Python,
420            tests: vec![GeneratedTest {
421                id: "BC-001".to_string(),
422                name: "Test".to_string(),
423                category: "boundary".to_string(),
424                points: 4,
425                severity: TestSeverity::High,
426                code: "    assert True".to_string(),
427            }],
428            total_points: 4,
429        };
430        let code = suite.to_code();
431        // Should have proper Python function name
432        assert!(code.contains("def test_bc_001"));
433        // Should have docstring
434        assert!(code.contains("BC-001: Test"));
435    }
436
437    // =====================================================================
438    // Coverage: generate_from_spec and generate_with_points
439    // =====================================================================
440
441    #[test]
442    fn test_generate_from_spec_with_file() {
443        let dir = tempfile::TempDir::new().expect("tempdir creation failed");
444        let spec_file = dir.path().join("test-spec.md");
445        std::fs::write(
446            &spec_file,
447            r#"# My Test Spec
448module: my_module
449
450## Requirements
451- MUST handle empty input gracefully
452- SHOULD return error on invalid data
453- The function MUST NOT panic on any input
454
455## Functions
456fn process_data(input: &[u8]) -> Result<Vec<u8>, Error>
457
458## Types
459struct DataProcessor { buffer: Vec<u8> }
460"#,
461        )
462        .expect("unexpected failure");
463
464        let engine = FalsifyEngine::new();
465        let suite = engine
466            .generate_from_spec(&spec_file, TargetLanguage::Rust)
467            .expect("unexpected failure");
468
469        assert_eq!(suite.spec_name, "test-spec");
470        assert_eq!(suite.language, TargetLanguage::Rust);
471        assert!(!suite.tests.is_empty());
472        assert_eq!(suite.total_points, 100);
473
474        // Verify generated code works
475        let code = suite.to_code();
476        assert!(code.contains("test-spec"));
477        assert!(code.contains("my_module"));
478    }
479
480    #[test]
481    fn test_generate_from_spec_python() {
482        let dir = tempfile::TempDir::new().expect("tempdir creation failed");
483        let spec_file = dir.path().join("py-spec.md");
484        std::fs::write(
485            &spec_file,
486            "module: test_mod\n- MUST handle empty input\n- SHOULD validate",
487        )
488        .expect("unexpected failure");
489
490        let engine = FalsifyEngine::new();
491        let suite = engine
492            .generate_from_spec(&spec_file, TargetLanguage::Python)
493            .expect("unexpected failure");
494
495        assert_eq!(suite.language, TargetLanguage::Python);
496        let code = suite.to_code();
497        assert!(code.contains("pytest"));
498        assert!(code.contains("hypothesis"));
499    }
500
501    #[test]
502    fn test_generate_from_spec_nonexistent_file() {
503        let engine = FalsifyEngine::new();
504        let result = engine
505            .generate_from_spec(std::path::Path::new("/nonexistent/file.md"), TargetLanguage::Rust);
506        assert!(result.is_err());
507    }
508
509    #[test]
510    fn test_generate_with_points() {
511        let dir = tempfile::TempDir::new().expect("tempdir creation failed");
512        let spec_file = dir.path().join("points-spec.md");
513        std::fs::write(
514            &spec_file,
515            "module: scaled_module\n- MUST work correctly\n- SHOULD be fast",
516        )
517        .expect("unexpected failure");
518
519        let engine = FalsifyEngine::new();
520        let suite = engine
521            .generate_with_points(&spec_file, TargetLanguage::Rust, 50)
522            .expect("unexpected failure");
523
524        assert_eq!(suite.spec_name, "points-spec");
525        assert_eq!(suite.total_points, 50);
526        assert!(!suite.tests.is_empty());
527    }
528
529    #[test]
530    fn test_generate_with_points_200() {
531        let dir = tempfile::TempDir::new().expect("tempdir creation failed");
532        let spec_file = dir.path().join("large-spec.md");
533        std::fs::write(&spec_file, "module: large_mod\n- MUST handle edge cases")
534            .expect("fs write failed");
535
536        let engine = FalsifyEngine::new();
537        let suite = engine
538            .generate_with_points(&spec_file, TargetLanguage::Python, 200)
539            .expect("unexpected failure");
540
541        assert_eq!(suite.total_points, 200);
542        assert_eq!(suite.language, TargetLanguage::Python);
543        let code = suite.to_code();
544        assert!(code.contains("pytest"));
545        assert!(!suite.tests.is_empty());
546    }
547
548    #[test]
549    fn test_generate_with_points_100_no_scaling() {
550        let dir = tempfile::TempDir::new().expect("tempdir creation failed");
551        let spec_file = dir.path().join("same-spec.md");
552        std::fs::write(&spec_file, "module: same_mod\n- MUST be tested").expect("fs write failed");
553
554        let engine = FalsifyEngine::new();
555        let suite = engine
556            .generate_with_points(&spec_file, TargetLanguage::Rust, 100)
557            .expect("unexpected failure");
558
559        // 100 points = default, no scaling needed
560        assert_eq!(suite.total_points, 100);
561    }
562
563    #[test]
564    fn test_generate_with_points_nonexistent_file() {
565        let engine = FalsifyEngine::new();
566        let result = engine.generate_with_points(
567            std::path::Path::new("/nonexistent/spec.md"),
568            TargetLanguage::Rust,
569            50,
570        );
571        assert!(result.is_err());
572    }
573
574    // =====================================================================
575    // Coverage: GeneratedSuite code generation with multiple categories
576    // =====================================================================
577
578    #[test]
579    fn test_rust_code_multiple_categories_same_category() {
580        let suite = GeneratedSuite {
581            spec_name: "multi".to_string(),
582            language: TargetLanguage::Rust,
583            tests: vec![
584                GeneratedTest {
585                    id: "BC-001".to_string(),
586                    name: "First boundary".to_string(),
587                    category: "boundary".to_string(),
588                    points: 4,
589                    severity: TestSeverity::High,
590                    code: "// bc1".to_string(),
591                },
592                GeneratedTest {
593                    id: "BC-002".to_string(),
594                    name: "Second boundary".to_string(),
595                    category: "boundary".to_string(),
596                    points: 4,
597                    severity: TestSeverity::Medium,
598                    code: "// bc2".to_string(),
599                },
600            ],
601            total_points: 8,
602        };
603        let code = suite.to_code();
604        // Category header should appear only once
605        let boundary_count = code.matches("BOUNDARY").count();
606        assert_eq!(boundary_count, 1, "BOUNDARY header should appear once");
607        assert!(code.contains("BC-001"));
608        assert!(code.contains("BC-002"));
609        assert!(code.contains("Points: 4"));
610    }
611
612    #[test]
613    fn test_python_code_multiple_categories() {
614        let suite = GeneratedSuite {
615            spec_name: "pytest".to_string(),
616            language: TargetLanguage::Python,
617            tests: vec![
618                GeneratedTest {
619                    id: "BC-001".to_string(),
620                    name: "Boundary".to_string(),
621                    category: "boundary".to_string(),
622                    points: 4,
623                    severity: TestSeverity::High,
624                    code: "    pass".to_string(),
625                },
626                GeneratedTest {
627                    id: "INV-001".to_string(),
628                    name: "Invariant".to_string(),
629                    category: "invariant".to_string(),
630                    points: 5,
631                    severity: TestSeverity::Critical,
632                    code: "    pass".to_string(),
633                },
634                GeneratedTest {
635                    id: "INV-002".to_string(),
636                    name: "Invariant2".to_string(),
637                    category: "invariant".to_string(),
638                    points: 5,
639                    severity: TestSeverity::High,
640                    code: "    pass".to_string(),
641                },
642            ],
643            total_points: 14,
644        };
645        let code = suite.to_code();
646        assert!(code.contains("BOUNDARY"));
647        assert!(code.contains("INVARIANT"));
648        assert!(code.contains("def test_bc_001"));
649        assert!(code.contains("def test_inv_001"));
650        assert!(code.contains("def test_inv_002"));
651        // Check docstrings
652        assert!(code.contains("Points: 4"));
653        assert!(code.contains("Points: 5"));
654    }
655
656    #[test]
657    fn test_suite_summary_multiple_categories() {
658        let suite = GeneratedSuite {
659            spec_name: "summary-test".to_string(),
660            language: TargetLanguage::Rust,
661            tests: vec![
662                GeneratedTest {
663                    id: "BC-001".to_string(),
664                    name: "T1".to_string(),
665                    category: "boundary".to_string(),
666                    points: 4,
667                    severity: TestSeverity::High,
668                    code: String::new(),
669                },
670                GeneratedTest {
671                    id: "NUM-001".to_string(),
672                    name: "T2".to_string(),
673                    category: "numerical".to_string(),
674                    points: 7,
675                    severity: TestSeverity::High,
676                    code: String::new(),
677                },
678                GeneratedTest {
679                    id: "NUM-002".to_string(),
680                    name: "T3".to_string(),
681                    category: "numerical".to_string(),
682                    points: 6,
683                    severity: TestSeverity::Medium,
684                    code: String::new(),
685                },
686            ],
687            total_points: 17,
688        };
689        let summary = suite.summary();
690        assert_eq!(summary.total_tests, 3);
691        assert_eq!(summary.total_points, 17);
692        assert_eq!(*summary.points_by_category.get("boundary").expect("key not found"), 4);
693        assert_eq!(*summary.points_by_category.get("numerical").expect("key not found"), 13);
694    }
695
696    #[test]
697    fn test_generated_suite_empty_tests() {
698        let suite = GeneratedSuite {
699            spec_name: "empty".to_string(),
700            language: TargetLanguage::Rust,
701            tests: vec![],
702            total_points: 0,
703        };
704        let code = suite.to_code();
705        assert!(code.contains("empty"));
706        assert!(code.contains("Total Points: 0"));
707        let summary = suite.summary();
708        assert_eq!(summary.total_tests, 0);
709        assert!(summary.points_by_category.is_empty());
710    }
711
712    #[test]
713    fn test_engine_template_accessor() {
714        let engine = FalsifyEngine::new();
715        let template = engine.template();
716        assert_eq!(template.total_points(), 100);
717        assert!(!template.categories.is_empty());
718        // Test get_category
719        assert!(template.get_category("boundary").is_some());
720        assert!(template.get_category("nonexistent").is_none());
721    }
722}
batuta/oracle/falsify/mod.rs

batuta/oracle/falsify/
mod.rs
