speccoder_core/
strength.rs

1//! Acceptance strength: how well a task's acceptance
2//! set tells right from wrong.
3//!
4//! A red → green acceptance test shows the task's work made it pass; it does
5//! not show the test would have failed a wrong implementation. After a
6//! `verified` verdict, wrong candidates — mutants of the changed code and
7//! model-written near misses — are run against the set: the share it rejects
8//! is its strength. Spec-permitted alternatives are run too: one the set
9//! rejects is a test that encodes a choice the spec did not make, and only a
10//! person can rule on it.
11//!
12//! These are the rules; running candidates is the runtime's.
13
14use serde::{Deserialize, Serialize};
15
16/// What a candidate run showed.
17#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
18#[serde(rename_all = "snake_case")]
19pub enum Outcome {
20    /// The acceptance run failed on an assertion (or a symbol it needs is gone).
21    Rejected,
22    /// The acceptance run passed: the set accepts this wrong implementation.
23    Survived,
24    /// The run hit its deadline: a detectable failure, counted as rejected.
25    Timeout,
26    /// It did not build or the tests could not be collected: not judged.
27    Unviable,
28}
29
30impl Outcome {
31    /// Counted as rejected.
32    #[must_use]
33    pub const fn rejects(self) -> bool {
34        matches!(self, Self::Rejected | Self::Timeout)
35    }
36
37    /// Judged at all.
38    #[must_use]
39    pub const fn judged(self) -> bool {
40        !matches!(self, Self::Unviable)
41    }
42}
43
44/// A model-written near miss: plausible, and wrong about one scenario.
45#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
46pub struct NearMiss {
47    /// `near-miss-1`, …
48    pub id: String,
49    /// The scenario or requirement it gets wrong.
50    pub scenario: String,
51    /// What it gets wrong, in one line (the author's claim).
52    pub rationale: String,
53    /// Against the visible set.
54    pub visible: Outcome,
55    /// Against the visible and the held-out set together.
56    pub combined: Outcome,
57}
58
59/// Mutants of the changed code, run against the acceptance command only.
60#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
61pub struct Mutants {
62    /// The tool.
63    pub tool: String,
64    /// Caught by an assertion.
65    pub caught: u32,
66    /// Not caught.
67    pub missed: u32,
68    /// Caught by the deadline.
69    pub timeout: u32,
70    /// Did not build.
71    pub unviable: u32,
72    /// The missed ones, as the tool names them.
73    pub survivors: Vec<String>,
74}
75
76/// What a spec-permitted alternative showed.
77#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
78#[serde(rename_all = "snake_case")]
79pub enum PermittedOutcome {
80    /// The set accepts it, as it should.
81    Accepted,
82    /// The set rejects it on an assertion: probably over-strict.
83    Rejected,
84    /// Rejected, and a person ruled the tests right.
85    Ruled,
86    /// Timed out: no finding (a hang is not an assertion).
87    Timeout,
88    /// Did not build: no finding.
89    Unviable,
90}
91
92/// An alternative the author says the spec allows.
93#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
94pub struct Permitted {
95    /// `permitted-1`, …
96    pub id: String,
97    /// The scenario whose open choice it takes differently.
98    pub scenario: String,
99    /// The freedom it uses (the author's claim).
100    pub rationale: String,
101    /// `sha256:<hex>` of its files: what a ruling is bound to.
102    pub digest: String,
103    /// Against the visible and held-out set together.
104    pub outcome: PermittedOutcome,
105    /// The failing run's last lines, when rejected.
106    #[serde(default, skip_serializing_if = "Option::is_none")]
107    pub failing: Option<String>,
108    /// Where its patch is kept for a person.
109    #[serde(default, skip_serializing_if = "Option::is_none")]
110    pub patch: Option<String>,
111}
112
113/// How far a measurement got.
114#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
115#[serde(rename_all = "snake_case")]
116pub enum State {
117    /// Everything planned was run.
118    Measured,
119    /// The budget ran out: what ran is recorded.
120    Partial,
121    /// Nothing could be judged (no tool, no variant, an error).
122    NotMeasured,
123    /// The change touches no source file.
124    NotApplicable,
125}
126
127/// Rejected of judged, for one source.
128#[derive(
129    Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema,
130)]
131pub struct Counts {
132    /// Rejected (timeouts included).
133    pub rejected: u32,
134    /// Judged (unviable excluded).
135    pub judged: u32,
136}
137
138/// One set's strength.
139#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, schemars::JsonSchema)]
140pub struct Strength {
141    /// How far it got.
142    pub state: State,
143    /// Rejected / judged over every source; `None` when nothing was judged.
144    pub share: Option<f64>,
145    /// Mutants; `None` when not run.
146    pub mutants: Option<Counts>,
147    /// Near misses.
148    pub near_misses: Counts,
149    /// Candidates that did not build.
150    pub unviable: u32,
151    /// What got through: mutant names and near-miss ids with their scenario.
152    pub survivors: Vec<String>,
153}
154
155/// The `min_strength` gate.
156#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
157#[serde(rename_all = "snake_case")]
158pub enum Gate {
159    /// No `min_strength`.
160    Off,
161    /// Met at once.
162    Met,
163    /// Met after one regeneration of the acceptance tests.
164    RegeneratedMet,
165    /// Not met, or not measurable.
166    Failed,
167}
168
169/// A person's ruling that an over-strict finding was wrong: the tests were
170/// right, the alternative is not what the spec allows.
171#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
172pub struct Ruling {
173    /// The alternative's digest.
174    pub digest: String,
175    /// Its scenario.
176    pub scenario: String,
177    /// Who ruled.
178    pub by: String,
179    /// Why.
180    pub reason: String,
181}
182
183/// The acceptance set's strength, as recorded with a verdict.
184#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, schemars::JsonSchema)]
185pub struct AcceptanceStrength {
186    /// The visible set alone.
187    pub visible: Strength,
188    /// Visible and held-out together: what the verdict rests on (equal to
189    /// `visible` without held-out tests). Mutants run with the visible
190    /// command only, so its mutant counts are a lower bound.
191    pub combined: Strength,
192    /// A held-out set was overlaid.
193    pub held_out: bool,
194    /// The mutants.
195    #[serde(default, skip_serializing_if = "Option::is_none")]
196    pub mutants: Option<Mutants>,
197    /// The near misses.
198    pub near_misses: Vec<NearMiss>,
199    /// The permitted alternatives.
200    pub permitted: Vec<Permitted>,
201    /// Rulings that applied.
202    #[serde(default, skip_serializing_if = "Vec::is_empty")]
203    pub rulings: Vec<Ruling>,
204    /// The gate.
205    pub gate: Gate,
206    /// Why not (fully) measured, or what was dropped.
207    #[serde(default, skip_serializing_if = "Vec::is_empty")]
208    pub notes: Vec<String>,
209    /// The near-miss call's run.
210    #[serde(default, skip_serializing_if = "Option::is_none")]
211    pub run_id: Option<String>,
212    /// The near-miss call's tokens.
213    #[serde(default, skip_serializing_if = "Option::is_none")]
214    pub usage: Option<serde_json::Value>,
215    /// How long the stage took.
216    pub duration_ms: u64,
217}
218
219/// Which set.
220#[derive(Debug, Clone, Copy, PartialEq, Eq)]
221pub enum Set {
222    /// The visible tests.
223    Visible,
224    /// Visible and held-out.
225    Combined,
226}
227
228/// One set's strength from what ran. `partial`: the budget ran out.
229#[must_use]
230pub fn strength(
231    mutants: Option<&Mutants>,
232    near_misses: &[NearMiss],
233    set: Set,
234    partial: bool,
235) -> Strength {
236    let mut unviable = 0;
237    let mut survivors = Vec::new();
238    let mutant_counts = mutants.map(|m| {
239        unviable += m.unviable;
240        survivors.extend(m.survivors.iter().cloned());
241        Counts {
242            rejected: m.caught + m.timeout,
243            judged: m.caught + m.timeout + m.missed,
244        }
245    });
246    let mut near = Counts::default();
247    for n in near_misses {
248        let o = match set {
249            Set::Visible => n.visible,
250            Set::Combined => n.combined,
251        };
252        if !o.judged() {
253            unviable += 1;
254            continue;
255        }
256        near.judged += 1;
257        if o.rejects() {
258            near.rejected += 1;
259        } else {
260            survivors.push(format!("{} ({}: {})", n.id, n.scenario, n.rationale));
261        }
262    }
263    let rejected = mutant_counts.map_or(0, |c| c.rejected) + near.rejected;
264    let judged = mutant_counts.map_or(0, |c| c.judged) + near.judged;
265    let share = (judged > 0).then(|| f64::from(rejected) / f64::from(judged));
266    let state = match (share, partial) {
267        (None, _) => State::NotMeasured,
268        (Some(_), true) => State::Partial,
269        (Some(_), false) => State::Measured,
270    };
271    Strength {
272        state,
273        share,
274        mutants: mutant_counts,
275        near_misses: near,
276        unviable,
277        survivors,
278    }
279}
280
281/// A strength that measured nothing, with its state.
282#[must_use]
283pub fn unmeasured(state: State) -> Strength {
284    Strength {
285        state,
286        share: None,
287        mutants: None,
288        near_misses: Counts::default(),
289        unviable: 0,
290        survivors: Vec::new(),
291    }
292}
293
294/// The alternatives the set rejected on an assertion and nobody ruled on:
295/// probably over-strict tests.
296#[must_use]
297pub fn over_strict(permitted: &[Permitted]) -> Vec<&Permitted> {
298    permitted
299        .iter()
300        .filter(|p| p.outcome == PermittedOutcome::Rejected)
301        .collect()
302}
303
304/// The gate for `min`. Only a full measurement can meet it: no evidence is
305/// not good evidence, and a partial one never satisfies it.
306#[must_use]
307pub fn gate(min: Option<f64>, s: &Strength, regenerated: bool) -> Gate {
308    let Some(min) = min else {
309        return Gate::Off;
310    };
311    match (s.state, s.share) {
312        (State::Measured, Some(share)) if share + f64::EPSILON >= min => {
313            if regenerated {
314                Gate::RegeneratedMet
315            } else {
316                Gate::Met
317            }
318        }
319        _ => Gate::Failed,
320    }
321}
322
323/// `0.83 mutants=9/11 near-miss=1/2 unviable=2`, or `not measured (…)`.
324#[must_use]
325pub fn summary(s: &Strength, why: Option<&str>) -> String {
326    let why = why.map(|w| format!(" ({w})")).unwrap_or_default();
327    match (s.state, s.share) {
328        (State::NotApplicable, _) => format!("not applicable{why}"),
329        (_, None) => format!("not measured{why}"),
330        (state, Some(share)) => {
331            let mut out = format!("{share:.2}");
332            if let Some(m) = s.mutants {
333                out.push_str(&format!(" mutants={}/{}", m.rejected, m.judged));
334            }
335            out.push_str(&format!(
336                " near-miss={}/{}",
337                s.near_misses.rejected, s.near_misses.judged
338            ));
339            if s.unviable > 0 {
340                out.push_str(&format!(" unviable={}", s.unviable));
341            }
342            if state == State::Partial {
343                out.push_str(" partial");
344            }
345            out
346        }
347    }
348}
349
350/// The trailers of a verified tick: `Acceptance-Strength:` (the combined
351/// set) and, with held-out tests, `Acceptance-Strength-Visible:`.
352#[must_use]
353pub fn trailers(a: &AcceptanceStrength) -> String {
354    let why = a.notes.first().map(String::as_str);
355    let mut out = format!("Acceptance-Strength: {}\n", summary(&a.combined, why));
356    if a.held_out {
357        out.push_str(&format!(
358            "Acceptance-Strength-Visible: {}\n",
359            summary(&a.visible, None)
360        ));
361    }
362    for r in &a.rulings {
363        out.push_str(&format!(
364            "Ruling: tests-right {} {} by {} — {}\n",
365            r.scenario, r.digest, r.by, r.reason
366        ));
367    }
368    out
369}
370
371#[cfg(test)]
372mod tests {
373    use super::*;
374    use proptest::prelude::*;
375
376    fn near(id: &str, visible: Outcome, combined: Outcome) -> NearMiss {
377        NearMiss {
378            id: id.to_owned(),
379            scenario: "US1-2".to_owned(),
380            rationale: "ignores the second example".to_owned(),
381            visible,
382            combined,
383        }
384    }
385
386    fn mutants(caught: u32, missed: u32) -> Mutants {
387        Mutants {
388            tool: "cargo-mutants".to_owned(),
389            caught,
390            missed,
391            timeout: 1,
392            unviable: 2,
393            survivors: (0..missed).map(|i| format!("src/lib.rs:{i}")).collect(),
394        }
395    }
396
397    #[test]
398    fn the_share_pools_sources_and_names_survivors() {
399        let m = mutants(8, 2);
400        let n = [
401            near("near-miss-1", Outcome::Rejected, Outcome::Rejected),
402            near("near-miss-2", Outcome::Survived, Outcome::Rejected),
403            near("near-miss-3", Outcome::Unviable, Outcome::Unviable),
404        ];
405        let v = strength(Some(&m), &n, Set::Visible, false);
406        // 9 of 11 mutants (timeout counts), 1 of 2 near misses.
407        assert_eq!(
408            v.mutants,
409            Some(Counts {
410                rejected: 9,
411                judged: 11
412            })
413        );
414        assert_eq!(
415            v.near_misses,
416            Counts {
417                rejected: 1,
418                judged: 2
419            }
420        );
421        assert_eq!(v.unviable, 3);
422        assert!((v.share.unwrap() - 10.0 / 13.0).abs() < 1e-9);
423        assert!(
424            v.survivors
425                .iter()
426                .any(|s| s.starts_with("near-miss-2 (US1-2"))
427        );
428        let c = strength(Some(&m), &n, Set::Combined, false);
429        assert!(c.share > v.share);
430        assert_eq!(
431            summary(&v, None),
432            "0.77 mutants=9/11 near-miss=1/2 unviable=3"
433        );
434    }
435
436    #[test]
437    fn nothing_judged_is_not_measured_and_meets_no_gate() {
438        let s = strength(
439            None,
440            &[near("n", Outcome::Unviable, Outcome::Unviable)],
441            Set::Visible,
442            false,
443        );
444        assert_eq!(s.state, State::NotMeasured);
445        assert_eq!(gate(Some(0.0), &s, false), Gate::Failed);
446        assert_eq!(gate(None, &s, false), Gate::Off);
447        assert_eq!(
448            summary(&s, Some("no mutation tool")),
449            "not measured (no mutation tool)"
450        );
451        assert_eq!(
452            summary(&unmeasured(State::NotApplicable), Some("no source change")),
453            "not applicable (no source change)"
454        );
455    }
456
457    #[test]
458    fn the_gate_takes_only_a_full_measurement() {
459        let n = [near("n", Outcome::Rejected, Outcome::Rejected)];
460        let full = strength(None, &n, Set::Combined, false);
461        let partial = strength(None, &n, Set::Combined, true);
462        assert_eq!(gate(Some(0.8), &full, false), Gate::Met);
463        assert_eq!(gate(Some(0.8), &full, true), Gate::RegeneratedMet);
464        assert_eq!(gate(Some(0.8), &partial, false), Gate::Failed);
465        assert!(summary(&partial, None).ends_with(" partial"));
466    }
467
468    #[test]
469    fn only_an_unruled_assertion_rejection_is_over_strict() {
470        let p = |id: &str, outcome| Permitted {
471            id: id.to_owned(),
472            scenario: "US2-1".to_owned(),
473            rationale: "another order".to_owned(),
474            digest: "sha256:ab".to_owned(),
475            outcome,
476            failing: None,
477            patch: None,
478        };
479        let all = [
480            p("a", PermittedOutcome::Accepted),
481            p("b", PermittedOutcome::Rejected),
482            p("c", PermittedOutcome::Ruled),
483            p("d", PermittedOutcome::Timeout),
484            p("e", PermittedOutcome::Unviable),
485        ];
486        let found: Vec<&str> = over_strict(&all).iter().map(|p| p.id.as_str()).collect();
487        assert_eq!(found, ["b"]);
488    }
489
490    #[test]
491    fn trailers_carry_both_sets_and_rulings() {
492        let n = [near("n", Outcome::Survived, Outcome::Rejected)];
493        let a = AcceptanceStrength {
494            visible: strength(None, &n, Set::Visible, false),
495            combined: strength(None, &n, Set::Combined, false),
496            held_out: true,
497            mutants: None,
498            near_misses: n.to_vec(),
499            permitted: Vec::new(),
500            rulings: vec![Ruling {
501                digest: "sha256:ab".to_owned(),
502                scenario: "US2-1".to_owned(),
503                by: "Dana".to_owned(),
504                reason: "order is specified in the plan".to_owned(),
505            }],
506            gate: Gate::Off,
507            notes: Vec::new(),
508            run_id: None,
509            usage: None,
510            duration_ms: 1,
511        };
512        assert_eq!(
513            trailers(&a),
514            "Acceptance-Strength: 1.00 near-miss=1/1\nAcceptance-Strength-Visible: 0.00 near-miss=0/1\nRuling: tests-right US2-1 sha256:ab by Dana — order is specified in the plan\n"
515        );
516    }
517
518    fn outcome() -> impl Strategy<Value = Outcome> {
519        prop_oneof![
520            Just(Outcome::Rejected),
521            Just(Outcome::Survived),
522            Just(Outcome::Timeout),
523            Just(Outcome::Unviable)
524        ]
525    }
526
527    proptest! {
528        #[test]
529        fn share_is_a_share_and_moves_the_right_way(
530            outcomes in proptest::collection::vec(outcome(), 0..12),
531            caught in 0u32..20, missed in 0u32..20,
532        ) {
533            let m = Mutants { caught, missed, ..Mutants::default() };
534            let n: Vec<NearMiss> = outcomes.iter().enumerate()
535                .map(|(i, o)| near(&format!("n{i}"), *o, *o)).collect();
536            let s = strength(Some(&m), &n, Set::Visible, false);
537            if let Some(share) = s.share {
538                prop_assert!((0.0..=1.0).contains(&share));
539            }
540            // An unviable candidate changes nothing; a rejected one never lowers it.
541            let mut more = n.clone();
542            more.push(near("u", Outcome::Unviable, Outcome::Unviable));
543            prop_assert_eq!(strength(Some(&m), &more, Set::Visible, false).share, s.share);
544            let mut more = n.clone();
545            more.push(near("r", Outcome::Rejected, Outcome::Rejected));
546            let after = strength(Some(&m), &more, Set::Visible, false).share.unwrap_or(0.0);
547            prop_assert!(after + 1e-12 >= s.share.unwrap_or(0.0));
548            // A partial measurement meets no gate.
549            prop_assert_eq!(gate(Some(0.0), &strength(Some(&m), &n, Set::Visible, true), false), Gate::Failed);
550        }
551    }
552}
speccoder_core/strength.rs

speccoder_core/
strength.rs
