llmlint 0.4.2

LLM-as-judge linter: enforce code-quality checks deterministic linters can't express, by driving real coding harnesses through oneharness.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
//! Render the judge's system prompt from a (user-customizable) template.
//!
//! Templates are [minijinja] (Jinja2-style). The context exposes `rules` (each
//! with `name`, `description`, and `rationale` — whether that rule wants a
//! justification — plus its compact `scope_mode` and `scope_files`), `files`
//! (the target paths), `file_rules` (per-file which
//! rules apply — the apply- or skip-list, whichever is shorter), `diffs`
//! (per-file changed-line diffs, present only under `--diff`), and `rationales`
//! (whether any rule in this batch wants one, to gate the rationale guidance
//! block). The built-in default template lives in `assets/default_template.md`
//! and is embedded via [`crate::io::assets`].

use serde::Serialize;

use crate::domain::applicability;
use crate::errors::{Error, Result};

/// One rule as presented to the judge in the rendered prompt.
#[derive(Debug, Clone, Serialize)]
pub struct RuleSpec {
    pub name: String,
    pub description: String,
    /// Whether this rule requires a `rationale` in the judge's verdict.
    pub rationale: bool,
    /// The relevance condition the judge must decide before evaluating this rule,
    /// or `None` for an always-evaluated rule. Exposed to the template so it can
    /// show the condition and gate the verdict on a `relevant` decision.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub relevance: Option<String>,
    /// Whether every violation of this rule must cite a concrete file + line.
    /// Exposed to the template so it can mark the rule and ask the judge to
    /// localize each violation.
    pub require_line_attribution: bool,
    /// The slash-relative files this rule applies to (a subset of the call's
    /// `files` union). Drives the per-file applicability context and the
    /// wrong-file validation.
    pub files: Vec<String>,
}

/// One target file's changed-line diff, shown to the judge under `--diff`. Kept
/// separate from `files` (a plain path list) so a custom template using
/// `{{ f }}` keeps working; the diff is also inlined per file in `file_rules`.
#[derive(Debug, Clone, Serialize)]
pub struct FileDiff {
    /// The file path (forward-slash form), matching its entry in `files`.
    pub file: String,
    /// The unified diff text for that file.
    pub diff: String,
}

/// One target file as the prompt presents it: its applicability (which rules to
/// evaluate against it) and, when `--diff` surfaced a change, its unified `diff`
/// inlined so the judge sees a changed file's rules and diff together.
#[derive(Serialize)]
struct FileEntry<'a> {
    file: &'a str,
    mode: applicability::Mode,
    rules: &'a [String],
    #[serde(skip_serializing_if = "Option::is_none")]
    diff: Option<&'a str>,
}

/// A rule plus its compact, computed scope presentation. Flattening preserves
/// the existing `r.name`, `r.files`, etc. custom-template interface.
#[derive(Serialize)]
struct RuleEntry<'a> {
    #[serde(flatten)]
    rule: &'a RuleSpec,
    scope_mode: applicability::Mode,
    scope_files: Vec<String>,
}

#[derive(Serialize)]
struct Context<'a> {
    rules: &'a [RuleEntry<'a>],
    files: &'a [String],
    /// Per-file presentation retained for custom templates and the correction
    /// prompt, with each changed file's inlined `diff`. The default template uses
    /// only the file and diff here; its applicability source is the compact scope
    /// computed beside each rule.
    file_rules: &'a [FileEntry<'a>],
    /// Per-file diffs (only files with changes), present under `--diff` and empty
    /// otherwise. Kept for custom templates; the default template inlines them
    /// per file via `file_rules`.
    diffs: &'a [FileDiff],
    /// True when at least one rule in this batch wants a rationale, so the
    /// template can show (or omit) the rationale guidance.
    rationales: bool,
    /// True when at least one rule in this batch carries a relevance condition,
    /// so the template can show (or omit) the relevance guidance.
    relevance: bool,
    /// True when at least one rule in this batch requires line attribution, so
    /// the template can show (or omit) the line-attribution guidance.
    line_attribution: bool,
}

/// Render `template` with the given rules, target file paths, and per-file
/// `diffs` (empty unless `--diff` is set). The per-file applicability
/// (`file_rules`) and compact per-rule scope are derived from each rule's
/// `files`. `rationales` gates the rationale guidance (true when any rule in this
/// batch wants one), `relevance`
/// gates the relevance guidance (true when any rule is conditional), and
/// `line_attribution` gates the line-attribution guidance (true when any rule
/// requires every violation to cite a file + line).
pub fn render(
    template: &str,
    rules: &[RuleSpec],
    files: &[String],
    diffs: &[FileDiff],
    rationales: bool,
    relevance: bool,
    line_attribution: bool,
) -> Result<String> {
    let rule_entries: Vec<RuleEntry> = rules
        .iter()
        .map(|rule| {
            let scope = applicability::per_rule(&rule.files, files);
            RuleEntry {
                rule,
                scope_mode: scope.mode,
                scope_files: scope.files,
            }
        })
        .collect();
    let pairs: Vec<(String, Vec<String>)> = rules
        .iter()
        .map(|r| (r.name.clone(), r.files.clone()))
        .collect();
    let applic = applicability::per_file(&pairs, files);
    // Pair each file's applicability with its diff (changed files only). Custom
    // templates retain the applicability fields; the default renders just the
    // file and its review material from this entry.
    let file_rules: Vec<FileEntry> = applic
        .iter()
        .map(|fr| FileEntry {
            file: &fr.file,
            mode: fr.mode,
            rules: &fr.rules,
            diff: diffs
                .iter()
                .find(|d| d.file == fr.file)
                .map(|d| d.diff.as_str()),
        })
        .collect();
    let mut env = minijinja::Environment::new();
    env.set_keep_trailing_newline(true);
    let ctx = Context {
        rules: &rule_entries,
        files,
        file_rules: &file_rules,
        diffs,
        rationales,
        relevance,
        line_attribution,
    };
    env.render_str(template, ctx)
        .map_err(|e| Error::Template(e.to_string()))
}

#[cfg(test)]
mod tests {
    use super::*;

    fn rules() -> Vec<RuleSpec> {
        vec![
            RuleSpec {
                name: "no_inline_sql".into(),
                description: "true when no SQL is inline; false otherwise.".into(),
                rationale: true,
                relevance: None,
                require_line_attribution: false,
                files: vec!["src/a.rs".into(), "src/b.rs".into()],
            },
            RuleSpec {
                name: "layered".into(),
                description: "true when layered.".into(),
                rationale: true,
                relevance: None,
                require_line_attribution: false,
                files: vec!["src/a.rs".into(), "src/b.rs".into()],
            },
        ]
    }

    #[test]
    fn renders_rules_and_files() {
        let tmpl = "Files:\n{% for f in files %}- {{ f }}\n{% endfor %}\
                    Rules:\n{% for r in rules %}* {{ r.name }}: {{ r.description }}\n{% endfor %}";
        let out = render(
            tmpl,
            &rules(),
            &["src/a.rs".into(), "src/b.rs".into()],
            &[],
            true,
            false,
            false,
        )
        .unwrap();
        assert!(out.contains("- src/a.rs"));
        assert!(out.contains("- src/b.rs"));
        assert!(out.contains("* no_inline_sql: true when no SQL is inline"));
        assert!(out.contains("* layered: true when layered."));
    }

    #[test]
    fn diffs_block_is_gated_and_renders_per_file() {
        // The `diffs` context block stays available for custom templates.
        let tmpl = "{% if diffs %}CHANGED\n{% for d in diffs %}{{ d.file }}:\n{{ d.diff }}\
                    {% endfor %}{% else %}WHOLE{% endif %}";
        // No diffs (the default): the gate is off.
        let off = render(
            tmpl,
            &rules(),
            &["src/a.rs".into()],
            &[],
            true,
            false,
            false,
        )
        .unwrap();
        assert!(off.contains("WHOLE"), "got: {off}");
        // With a diff: the block renders the file path and its diff text.
        let diffs = vec![FileDiff {
            file: "src/a.rs".into(),
            diff: "@@ -1 +1 @@\n-old\n+new\n".into(),
        }];
        let on = render(
            tmpl,
            &rules(),
            &["src/a.rs".into()],
            &diffs,
            true,
            false,
            false,
        )
        .unwrap();
        assert!(on.contains("CHANGED"), "got: {on}");
        assert!(on.contains("src/a.rs:"), "got: {on}");
        assert!(on.contains("+new"), "got: {on}");
    }

    #[test]
    fn file_rules_inline_the_diff_for_a_changed_file_only() {
        // Each file_rules entry carries its diff when changed, so the template can
        // show a changed file's rules and diff together; an unchanged file has none.
        let tmpl = "{% for fr in file_rules %}{{ fr.file }}\
                    {% if fr.diff %} DIFF[{{ fr.diff }}]{% endif %}\n{% endfor %}";
        let diffs = vec![FileDiff {
            file: "src/a.rs".into(),
            diff: "@@ -1 +1 @@\n+new\n".into(),
        }];
        let out = render(
            tmpl,
            &rules(),
            &["src/a.rs".into(), "src/b.rs".into()],
            &diffs,
            false,
            false,
            false,
        )
        .unwrap();
        assert!(
            out.contains("src/a.rs DIFF[@@ -1 +1 @@\n+new\n]"),
            "out:\n{out}"
        );
        // The unchanged file gets no inlined diff.
        assert!(out.contains("src/b.rs\n"), "out:\n{out}");
        assert!(!out.contains("src/b.rs DIFF"), "out:\n{out}");
    }

    #[test]
    fn rationales_flag_and_per_rule_rationale_are_in_scope() {
        let tmpl = "{% if rationales %}WANT{% else %}SKIP{% endif %}\n\
                    {% for r in rules %}{{ r.name }}={{ r.rationale }}\n{% endfor %}";
        let on = render(tmpl, &rules(), &[], &[], true, false, false).unwrap();
        assert!(on.contains("WANT"));
        assert!(on.contains("no_inline_sql=true"));
        let off = render(tmpl, &rules(), &[], &[], false, false, false).unwrap();
        assert!(off.contains("SKIP"));
    }

    #[test]
    fn relevance_flag_and_per_rule_condition_are_in_scope() {
        let tmpl = "{% if relevance %}GATE{% else %}NOGATE{% endif %}\n\
                    {% for r in rules %}{% if r.relevance %}{{ r.name }}: {{ r.relevance }}\n\
                    {% endif %}{% endfor %}";
        let mut rs = rules();
        rs[0].relevance = Some("the change touches SQL".into());
        let on = render(tmpl, &rs, &[], &[], true, true, false).unwrap();
        assert!(on.contains("GATE"));
        assert!(on.contains("no_inline_sql: the change touches SQL"));
        // The always-evaluated rule renders no condition line.
        assert!(!on.contains("layered:"));
        let off = render(tmpl, &rules(), &[], &[], true, false, false).unwrap();
        assert!(off.contains("NOGATE"));
    }

    #[test]
    fn line_attribution_flag_and_per_rule_marker_are_in_scope() {
        let tmpl = "{% if line_attribution %}LOCALIZE{% else %}ANYWHERE{% endif %}\n\
                    {% for r in rules %}{% if r.require_line_attribution %}{{ r.name }} pinned\n\
                    {% endif %}{% endfor %}";
        let mut rs = rules();
        rs[0].require_line_attribution = true;
        let on = render(tmpl, &rs, &[], &[], true, false, true).unwrap();
        assert!(on.contains("LOCALIZE"));
        assert!(on.contains("no_inline_sql pinned"));
        // The rule that doesn't require attribution renders no marker line.
        assert!(!on.contains("layered pinned"));
        let off = render(tmpl, &rules(), &[], &[], true, false, false).unwrap();
        assert!(off.contains("ANYWHERE"));
    }

    #[test]
    fn the_default_template_demands_completeness_for_every_flag_combination() {
        // Reporting every violation governs *every* rule, so the instruction must
        // render with each conditional block off — notably `line_attribution`,
        // whose section used to be the only place it appeared (so a batch with no
        // opted-in rule was told nothing about completeness).
        for rationales in [false, true] {
            for relevance in [false, true] {
                for line_attribution in [false, true] {
                    let mut rs = rules();
                    rs[0].rationale = rationales;
                    rs[0].relevance = relevance.then(|| "the change touches SQL".into());
                    rs[0].require_line_attribution = line_attribution;
                    let out = render(
                        crate::io::assets::DEFAULT_TEMPLATE,
                        &rs,
                        &["src/a.rs".into(), "src/b.rs".into()],
                        &[],
                        rationales,
                        relevance,
                        line_attribution,
                    )
                    .unwrap_or_else(|e| {
                        panic!("default template failed to render ({rationales}/{relevance}/{line_attribution}): {e}")
                    });
                    assert!(
                        out.contains("report **every** distinct violation"),
                        "no completeness instruction ({rationales}/{relevance}/{line_attribution}):\n{out}"
                    );
                    assert!(
                        out.contains("**Judge each rule independently.**"),
                        "no rule-independence instruction:\n{out}"
                    );
                    // The gated sections still track their flags.
                    assert_eq!(out.contains("## Line attribution"), line_attribution);
                    assert_eq!(out.contains("## Rationale"), rationales);
                    assert_eq!(out.contains("## Relevance"), relevance);

                    // The rationale precedes the verdict, so a violating rule is
                    // told to enumerate its sites before concluding; a holding one
                    // stays as terse as it ever was.
                    assert_eq!(
                        out.contains("When the property holds, keep it terse"),
                        rationales
                    );
                    assert_eq!(
                        out.contains("account for **every** site you found"),
                        rationales
                    );

                    // Line attribution governs citation, not completeness: an
                    // unmarked rule is let off the file+line requirement alone and
                    // still owes a complete list.
                    assert_eq!(
                        out.contains("Every violation must cite a `file` and `line`."),
                        line_attribution
                    );
                    assert_eq!(
                        out.contains("exempt from the citation requirement only"),
                        line_attribution
                    );
                    assert_eq!(
                        out.contains("list must still be complete"),
                        line_attribution
                    );
                }
            }
        }
    }

    #[test]
    fn file_rules_expose_per_file_applicability() {
        // Two rules with different file scopes: a.rs gets only_a, b.rs gets only_b.
        let rs = vec![
            RuleSpec {
                name: "only_a".into(),
                description: "d".into(),
                rationale: false,
                relevance: None,
                require_line_attribution: false,
                files: vec!["src/a.rs".into()],
            },
            RuleSpec {
                name: "only_b".into(),
                description: "d".into(),
                rationale: false,
                relevance: None,
                require_line_attribution: false,
                files: vec!["src/b.rs".into()],
            },
        ];
        let tmpl = "{% for fr in file_rules %}{{ fr.file }}:{{ fr.mode }}:\
                    {% for r in fr.rules %}{{ r }} {% endfor %}\n{% endfor %}";
        let out = render(
            tmpl,
            &rs,
            &["src/a.rs".into(), "src/b.rs".into()],
            &[],
            false,
            false,
            false,
        )
        .unwrap();
        assert!(out.contains("src/a.rs:include:only_a"), "out:\n{out}");
        assert!(out.contains("src/b.rs:include:only_b"), "out:\n{out}");
    }

    #[test]
    fn invalid_template_is_a_template_error() {
        let err = render("{% for x in %}", &rules(), &[], &[], true, false, false).unwrap_err();
        assert!(matches!(err, Error::Template(_)));
    }
}