vtcode_eval/
test_quality.rs

1//! Test-quality signal for agentic testing (Dan Luu `agentic-testing`).
2//!
3//! `pass@k` measures whether runs pass. This module measures whether the
4//! *tests themselves* are high-value: asymmetric/boundary coverage, structured
5//! property-based testing with shrinking, and avoidance of panic-only or
6//! identical-fixture traps.
7//!
8//! Heuristics are intentionally cheap string analyses over test sources so they
9//! can run in CI without executing mutations. They are a triage signal, not a
10//! proof of correctness.
11
12use serde::{Deserialize, Serialize};
13
14/// Aggregate facts about a test source file or inline test module.
15#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
16pub struct TestQualitySummary {
17    /// Number of test cases (`#[test]`-family attributes).
18    pub total_tests: u64,
19    /// Tests with no oracle (`assert*`, `prop_assert*`, `insta::*`).
20    pub panic_only_tests: u64,
21    /// Tests exercising asymmetric, boundary, order-swap, or reversal cases.
22    pub asymmetric_tests: u64,
23    /// Property tests using structured strategies (`prop_oneof`, `prop_compose`, `Just`, `strategy`).
24    pub structured_pbt_tests: u64,
25}
26
27impl TestQualitySummary {
28    /// Fraction of tests with no value oracle. Lower is better.
29    pub fn panic_only_ratio(&self) -> f64 {
30        if self.total_tests == 0 {
31            return 0.0;
32        }
33        self.panic_only_tests as f64 / self.total_tests as f64
34    }
35
36    /// Fraction of tests covering asymmetric/boundary behavior. Higher is better.
37    pub fn asymmetric_coverage(&self) -> f64 {
38        if self.total_tests == 0 {
39            return 0.0;
40        }
41        self.asymmetric_tests as f64 / self.total_tests as f64
42    }
43
44    /// Fraction of tests using structured PBT. Higher is better.
45    pub fn structured_pbt_ratio(&self) -> f64 {
46        if self.total_tests == 0 {
47            return 0.0;
48        }
49        self.structured_pbt_tests as f64 / self.total_tests as f64
50    }
51
52    /// Luu-inspired bar: mostly oracle-backed, some asymmetric coverage.
53    pub fn meets_agentic_testing_bar(&self) -> bool {
54        self.total_tests > 0 && self.panic_only_ratio() <= 0.5 && self.asymmetric_coverage() >= 0.2
55    }
56
57    /// Merge another summary into this aggregate.
58    pub fn merge(&mut self, other: &Self) {
59        self.total_tests = self.total_tests.saturating_add(other.total_tests);
60        self.panic_only_tests = self.panic_only_tests.saturating_add(other.panic_only_tests);
61        self.asymmetric_tests = self.asymmetric_tests.saturating_add(other.asymmetric_tests);
62        self.structured_pbt_tests = self.structured_pbt_tests.saturating_add(other.structured_pbt_tests);
63    }
64}
65
66/// Analyze a Rust test source string with cheap heuristics.
67pub fn analyze_test_source(source: &str) -> TestQualitySummary {
68    let mut summary = TestQualitySummary::default();
69    for block in split_test_blocks(source) {
70        summary.total_tests = summary.total_tests.saturating_add(1);
71        if is_panic_only(block) {
72            summary.panic_only_tests = summary.panic_only_tests.saturating_add(1);
73        }
74        if is_asymmetric(block) {
75            summary.asymmetric_tests = summary.asymmetric_tests.saturating_add(1);
76        }
77        if is_structured_pbt(block) {
78            summary.structured_pbt_tests = summary.structured_pbt_tests.saturating_add(1);
79        }
80    }
81    summary
82}
83
84fn split_test_blocks(source: &str) -> Vec<&str> {
85    let mut blocks = Vec::new();
86    let mut rest = source;
87    loop {
88        // Prefer `#[test]`-family markers so a `proptest!` header is never
89        // counted as its own test; fall back to bare `proptest!` only when no
90        // inner test attribute exists.
91        let attr_pos = ["#[test]", "#[tokio::test]", "#[rstest]"]
92            .iter()
93            .filter_map(|marker| rest.find(marker))
94            .min();
95        let pos = match attr_pos {
96            Some(pos) => pos,
97            None => match rest.find("proptest!") {
98                Some(pos) => pos,
99                None => break,
100            },
101        };
102        let next_attr = ["#[test]", "#[tokio::test]", "#[rstest]"]
103            .iter()
104            .filter_map(|marker| rest[pos + 1..].find(marker).map(|next| pos + 1 + next))
105            .min();
106        let next_prop = rest[pos + 1..].find("proptest!").map(|next| pos + 1 + next);
107        let next_test = match (next_attr, next_prop) {
108            (Some(attr), _) => Some(attr),
109            (None, prop) => prop,
110        };
111        let end = next_test.unwrap_or(rest.len());
112        blocks.push(&rest[pos..end]);
113        if next_test.is_none() {
114            break;
115        }
116        rest = &rest[end..];
117    }
118    blocks
119}
120
121fn is_panic_only(block: &str) -> bool {
122    // `prop_assume!` filters inputs and `panic!` fails explicitly; neither is a
123    // value oracle. `assert!`/`assert_*` covers `assert_*`/`prop_assert*` while
124    // avoiding prose matches like `assertion`.
125    !(block.contains("assert!") || block.contains("assert_") || block.contains("insta::") || block.contains("expect("))
126}
127
128fn is_asymmetric(block: &str) -> bool {
129    let lowered = block.to_ascii_lowercase();
130    if lowered.contains("both sides") {
131        return true;
132    }
133    // Match whole tokens so `bright`/`leftover`/`border` do not count as
134    // `right`/`left`/`order` coverage.
135    let tokens: Vec<&str> = lowered
136        .split(|character: char| !character.is_ascii_alphanumeric())
137        .filter(|token| !token.is_empty())
138        .collect();
139    const STEMS: &[&str] = &[
140        "asymmetr",
141        "boundary",
142        "boundaries",
143        "revers",
144        "swap",
145        "transpos",
146        "palindrom",
147        "identical",
148    ];
149    const WORDS: &[&str] = &["order", "left", "right"];
150    tokens
151        .iter()
152        .any(|token| STEMS.iter().any(|stem| token.contains(stem)) || WORDS.contains(token))
153}
154
155fn is_structured_pbt(block: &str) -> bool {
156    block.contains("prop_oneof")
157        || block.contains("prop_compose")
158        || block.contains("Just(")
159        || block.contains("strategy")
160        || block.contains("prop::")
161}
162
163#[cfg(test)]
164mod tests {
165    use super::*;
166
167    #[test]
168    fn counts_oracle_backed_and_asymmetric_tests() {
169        let source = r#"
170            #[test]
171            fn transposed_order_is_rejected_asymmetric_boundary() {
172                assert_ne!(vec!["a", "b"], vec!["b", "a"]);
173            }
174            #[test]
175            fn never_panics_smoke() {
176                let _value = vec![1, 2, 3].len();
177            }
178        "#;
179        let summary = analyze_test_source(source);
180        assert_eq!(summary.total_tests, 2);
181        assert_eq!(summary.panic_only_tests, 1);
182        assert_eq!(summary.asymmetric_tests, 1);
183        assert!((summary.panic_only_ratio() - 0.5).abs() < f64::EPSILON);
184        assert!((summary.asymmetric_coverage() - 0.5).abs() < f64::EPSILON);
185    }
186
187    #[test]
188    fn detects_structured_proptest_strategies() {
189        let source = r#"
190            proptest! {
191                #[test]
192                fn order_swap_uses_structured_strategy(a in prop::option::of(Just(1))) {
193                    prop_assert!(a.is_some());
194                }
195            }
196        "#;
197        let summary = analyze_test_source(source);
198        assert_eq!(summary.total_tests, 1);
199        assert_eq!(summary.panic_only_tests, 0);
200        assert_eq!(summary.structured_pbt_tests, 1);
201        assert!(summary.meets_agentic_testing_bar());
202    }
203
204    #[test]
205    fn empty_source_meets_no_bar_but_reports_zero_ratios() {
206        let summary = analyze_test_source("");
207        assert_eq!(summary.total_tests, 0);
208        assert_eq!(summary.panic_only_ratio(), 0.0);
209        assert_eq!(summary.asymmetric_coverage(), 0.0);
210        assert!(!summary.meets_agentic_testing_bar());
211    }
212
213    #[test]
214    fn prop_assume_and_panic_are_not_oracles() {
215        let assume_only = "#[test] fn filtered() { prop_assume!(true); }";
216        assert_eq!(analyze_test_source(assume_only).panic_only_tests, 1);
217        let panic_only = "#[test] fn todo() { panic!(\"todo\"); }";
218        assert_eq!(analyze_test_source(panic_only).panic_only_tests, 1);
219        let asserted = "#[test] fn checked() { assert!(true); }";
220        assert_eq!(analyze_test_source(asserted).panic_only_tests, 0);
221    }
222
223    #[test]
224    fn asymmetric_matching_uses_word_boundaries() {
225        assert_eq!(analyze_test_source("#[test] fn b() { let brightness = 1; assert!(true); }").asymmetric_tests, 0);
226        assert_eq!(analyze_test_source("#[test] fn b() { let leftover = 1; assert!(true); }").asymmetric_tests, 0);
227        assert_eq!(analyze_test_source("#[test] fn b() { fn render_border() {} assert!(true); }").asymmetric_tests, 0);
228        assert_eq!(
229            analyze_test_source("#[test] fn swapped() { assert_ne!(vec![\"a\"], vec![\"b\"]); }").asymmetric_tests,
230            1
231        );
232    }
233
234    #[test]
235    fn counts_tokio_tests() {
236        let source = "#[tokio::test] async fn async_case() { assert!(true); }";
237        assert_eq!(analyze_test_source(source).total_tests, 1);
238    }
239
240    #[test]
241    fn merge_sums_counters() {
242        let mut left = analyze_test_source("#[test] fn a() { assert!(true); }");
243        let right = analyze_test_source("#[test] fn b() { let _x = 1; }");
244        left.merge(&right);
245        assert_eq!(left.total_tests, 2);
246        assert_eq!(left.panic_only_tests, 1);
247    }
248}
vtcode_eval/test_quality.rs

vtcode_eval/
test_quality.rs
