llmlint 0.2.3

LLM-as-judge linter: enforce code-quality checks deterministic linters can't express, by driving real coding harnesses through oneharness.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109

//! Aggregate independent judges' verdicts for a rule into one outcome.

use std::collections::HashSet;

use crate::domain::verdict::{Outcome, RuleOutcome, RuleVerdict, Violation};

/// Tally the verdicts for a single rule. A rule **passes** only with a strict
/// majority of `holds = true` (a tie fails — conservative for a linter). On a
/// fail, violations from the failing judges are unioned and de-duplicated.
pub fn tally(name: &str, verdicts: &[RuleVerdict]) -> RuleOutcome {
    let total = verdicts.len() as u32;
    let votes_hold = verdicts.iter().filter(|v| v.holds).count() as u32;
    let passes = votes_hold * 2 > total;

    let violations = if passes {
        Vec::new()
    } else {
        dedup(
            verdicts
                .iter()
                .filter(|v| !v.holds)
                .flat_map(|v| v.violations.iter().cloned()),
        )
    };

    RuleOutcome {
        name: name.to_string(),
        outcome: if passes { Outcome::Pass } else { Outcome::Fail },
        votes_total: total,
        votes_hold,
        violations,
    }
}

fn dedup(items: impl Iterator<Item = Violation>) -> Vec<Violation> {
    let mut seen = HashSet::new();
    let mut out = Vec::new();
    for v in items {
        if seen.insert(v.dedup_key()) {
            out.push(v);
        }
    }
    out
}

#[cfg(test)]
mod tests {
    use super::*;

    fn v(holds: bool, msg: &str) -> RuleVerdict {
        RuleVerdict {
            holds,
            violations: if holds {
                vec![]
            } else {
                vec![Violation {
                    message: Some(msg.into()),
                    ..Default::default()
                }]
            },
        }
    }

    #[test]
    fn single_pass() {
        let o = tally("r", &[v(true, "")]);
        assert_eq!(o.outcome, Outcome::Pass);
        assert_eq!(o.votes_total, 1);
        assert_eq!(o.votes_hold, 1);
        assert!(o.violations.is_empty());
    }

    #[test]
    fn single_fail_keeps_violation() {
        let o = tally("r", &[v(false, "bad")]);
        assert_eq!(o.outcome, Outcome::Fail);
        assert_eq!(o.violations.len(), 1);
        assert_eq!(o.violations[0].message.as_deref(), Some("bad"));
    }

    #[test]
    fn majority_pass_three_judges() {
        let o = tally("r", &[v(true, ""), v(false, "x"), v(true, "")]);
        assert_eq!(o.outcome, Outcome::Pass);
        assert_eq!(o.votes_hold, 2);
        // Passing rule reports no violations even though a minority dissented.
        assert!(o.violations.is_empty());
    }

    #[test]
    fn majority_fail_three_judges_unions_violations() {
        let o = tally("r", &[v(false, "a"), v(true, ""), v(false, "b")]);
        assert_eq!(o.outcome, Outcome::Fail);
        assert_eq!(o.votes_hold, 1);
        assert_eq!(o.violations.len(), 2);
    }

    #[test]
    fn tie_fails() {
        let o = tally("r", &[v(true, ""), v(false, "x")]);
        assert_eq!(o.outcome, Outcome::Fail);
    }

    #[test]
    fn duplicate_violations_are_deduped() {
        let o = tally("r", &[v(false, "same"), v(false, "same")]);
        assert_eq!(o.violations.len(), 1);
    }
}