magi/
blind.rs

1//! Blindness: label assignment, attribution stripping, and leak detection.
2//!
3//! A judge that knows which model wrote a candidate stops grading the patch and
4//! starts voting on the model's reputation. Three things keep that from
5//! happening:
6//!
7//! 1. **Labels.** Candidates are presented as `A`/`B`/`C`, assigned by a seeded
8//!    shuffle, and each judge sees them in its own order so position carries no
9//!    signal either.
10//! 2. **Stripping.** Commit messages and candidate summaries lose their
11//!    attribution trailers — both at write time (a per-worktree `commit-msg`
12//!    hook) and at presentation time (this module). Belt and braces: the hook
13//!    can be bypassed with `--no-verify`, the presentation filter cannot.
14//! 3. **Leak detection.** The patch body is scanned for vendor-identifying
15//!    text. Blanket redaction there would corrupt the artifact under judgement,
16//!    so the policy is configurable and defaults to recording the leak.
17//!
18//! magi itself is the facilitator, which is the structural reason this works:
19//! there is no moderator agent that *could* leak an author, because the
20//! moderator is code that never learns anything it does not print.
21use crate::config::{Blind, LeakPolicy};
22use crate::rng::SplitMix64;
23
24/// A vendor token found in material shown to judges.
25#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
26pub struct Leak {
27    /// Where it was found, e.g. `candidate B patch`.
28    pub site: String,
29    /// The token, as configured.
30    pub token: String,
31    /// How many times it occurred.
32    pub count: usize,
33}
34
35/// Label for candidate index `i` after the seeded shuffle.
36///
37/// Returns one label per candidate: `labels[i]` is the label candidate `i` is
38/// presented under.
39pub fn assign_labels(n: usize, seed: u64) -> Vec<char> {
40    let mut pool: Vec<char> = (0..n).map(label_char).collect();
41    SplitMix64::new(seed).shuffle(&mut pool);
42    pool
43}
44
45/// `0 -> 'A'`, `25 -> 'Z'`, then wraps with a digit suffix scheme that is still
46/// unique but never reached in practice (`candidates` above 26 is nonsense).
47fn label_char(i: usize) -> char {
48    char::from(b'A' + (i % 26) as u8)
49}
50
51/// The order judge `j` sees the candidates in, as indices into the candidate
52/// list.
53pub fn presentation_order(n: usize, judge: usize, seed: u64) -> Vec<usize> {
54    let mut order: Vec<usize> = (0..n).collect();
55    SplitMix64::new(seed ^ crate::rng::fnv1a(&format!("judge-order-{judge}"))).shuffle(&mut order);
56    order
57}
58
59/// Drop every line containing one of `patterns` (case-insensitive substring).
60pub fn strip_attribution(text: &str, patterns: &[String]) -> String {
61    let mut out = String::with_capacity(text.len());
62    for line in text.lines() {
63        let lowered = ascii_lower(line);
64        if patterns
65            .iter()
66            .any(|p| !p.is_empty() && lowered.contains(&ascii_lower(p)))
67        {
68            continue;
69        }
70        out.push_str(line);
71        out.push('\n');
72    }
73    out
74}
75
76/// Count occurrences of each vendor token in `text`.
77pub fn scan(site: &str, text: &str, tokens: &[String]) -> Vec<Leak> {
78    let lowered = ascii_lower(text);
79    tokens
80        .iter()
81        .filter(|t| !t.is_empty())
82        .filter_map(|t| {
83            let count = lowered.matches(&ascii_lower(t)).count();
84            (count > 0).then(|| Leak {
85                site: site.to_owned(),
86                token: t.clone(),
87                count,
88            })
89        })
90        .collect()
91}
92
93/// Replace every vendor token with `[REDACTED]`, case-insensitively.
94pub fn redact(text: &str, tokens: &[String]) -> String {
95    const PLACEHOLDER: &str = "[REDACTED]";
96    let mut out = text.to_owned();
97    for t in tokens.iter().filter(|t| !t.is_empty()) {
98        let needle = ascii_lower(t);
99        // Scan the pre-replacement copy and build a new string, so the
100        // placeholder is never itself scanned. Replacing in place and
101        // restarting would loop forever on any token whose letters occur in
102        // `PLACEHOLDER` — "codex" and "cursor" both do.
103        let lowered = ascii_lower(&out);
104        let mut result = String::with_capacity(out.len());
105        let mut cursor = 0usize;
106        while let Some(rel) = lowered[cursor..].find(&needle) {
107            let at = cursor + rel;
108            result.push_str(&out[cursor..at]);
109            result.push_str(PLACEHOLDER);
110            cursor = at + t.len();
111        }
112        result.push_str(&out[cursor..]);
113        out = result;
114    }
115    out
116}
117
118/// Sanitize prose written by a candidate (commit messages, summaries).
119///
120/// Always strips *and* redacts: prose has no structural value to preserve, and
121/// it is where "Generated with X" actually shows up.
122pub fn sanitize_prose(text: &str, cfg: &Blind) -> String {
123    let stripped = strip_attribution(text, &cfg.strip_lines);
124    redact(&stripped, &cfg.vendor_tokens).trim().to_owned()
125}
126
127/// Apply the configured leak policy to a patch body.
128///
129/// Returns the text to show the judges plus everything found.
130pub fn sanitize_patch(site: &str, patch: &str, cfg: &Blind) -> (String, Vec<Leak>) {
131    let leaks = scan(site, patch, &cfg.vendor_tokens);
132    let text = match cfg.on_leak {
133        LeakPolicy::Redact => redact(patch, &cfg.vendor_tokens),
134        LeakPolicy::Warn | LeakPolicy::Fail => patch.to_owned(),
135    };
136    (text, leaks)
137}
138
139/// The `commit-msg` hook installed into every candidate worktree.
140///
141/// POSIX `sh` plus `sed`, which is what git runs hooks with on every platform
142/// magi supports, git-for-windows included. `sed` is given a case-insensitive
143/// expansion of each configured substring so the hook and
144/// [`strip_attribution`] agree on what counts as attribution.
145pub fn commit_msg_hook(patterns: &[String]) -> String {
146    let mut script = String::from(
147        "#!/bin/sh\n\
148         # Installed by magi. Candidate history must not name its author:\n\
149         # a judge that can read `Co-Authored-By:` is no longer blind.\n\
150         set -e\n\
151         msg=\"$1\"\n\
152         tmp=\"${msg}.magi\"\n\
153         sed \\\n",
154    );
155    for p in patterns.iter().filter(|p| !p.is_empty()) {
156        script.push_str(&format!("  -e '/{}/d' \\\n", sed_ci_pattern(p)));
157    }
158    script.push_str(
159        "  \"$msg\" > \"$tmp\"\n\
160         mv \"$tmp\" \"$msg\"\n",
161    );
162    script
163}
164
165/// Turn a literal substring into a case-insensitive basic-regex, escaping the
166/// metacharacters that matter inside a `sed` address.
167fn sed_ci_pattern(literal: &str) -> String {
168    let mut out = String::with_capacity(literal.len() * 4);
169    for ch in literal.chars() {
170        if ch.is_ascii_alphabetic() {
171            out.push('[');
172            out.push(ch.to_ascii_uppercase());
173            out.push(ch.to_ascii_lowercase());
174            out.push(']');
175        } else if matches!(ch, '.' | '*' | '[' | ']' | '^' | '$' | '\\' | '/') {
176            out.push('\\');
177            out.push(ch);
178        } else if ch == '\'' {
179            // Cannot appear inside the single-quoted sed expression.
180            out.push('.');
181        } else {
182            out.push(ch);
183        }
184    }
185    out
186}
187
188/// ASCII-only lowercase.
189///
190/// [`str::to_lowercase`] is Unicode-aware and can change a string's byte
191/// length, which would invalidate the offsets [`redact`] splices at. Folding
192/// only ASCII keeps every byte offset identical between the original and the
193/// lowered copy.
194fn ascii_lower(s: &str) -> String {
195    let mut out = String::with_capacity(s.len());
196    for b in s.chars() {
197        out.push(if b.is_ascii_uppercase() {
198            b.to_ascii_lowercase()
199        } else {
200            b
201        });
202    }
203    out
204}
205
206#[cfg(test)]
207mod tests {
208    use super::*;
209
210    fn cfg() -> Blind {
211        Blind::default()
212    }
213
214    #[test]
215    fn labels_are_a_permutation_and_stable_for_a_seed() {
216        let a = assign_labels(3, 99);
217        let b = assign_labels(3, 99);
218        assert_eq!(a, b);
219        let mut sorted = a.clone();
220        sorted.sort_unstable();
221        assert_eq!(sorted, ['A', 'B', 'C']);
222    }
223
224    #[test]
225    fn each_judge_gets_its_own_presentation_order() {
226        let orders: Vec<Vec<usize>> = (0..3).map(|j| presentation_order(3, j, 5)).collect();
227        for o in &orders {
228            let mut s = o.clone();
229            s.sort_unstable();
230            assert_eq!(s, [0, 1, 2]);
231        }
232        assert!(
233            orders.iter().any(|o| *o != orders[0]),
234            "three judges should not all see the same order: {orders:?}"
235        );
236    }
237
238    #[test]
239    fn trailers_are_stripped_case_insensitively() {
240        let msg = "Add retry\n\nBody text.\nco-authored-by: Claude <noreply@anthropic.com>\n\
241                   Co-Authored-By: Someone\nGenerated with the thing\nkeep me\n";
242        let out = strip_attribution(msg, &cfg().strip_lines);
243        assert!(out.contains("Add retry"));
244        assert!(out.contains("keep me"));
245        assert!(!out.to_lowercase().contains("co-authored-by"));
246        assert!(!out.contains("Generated with"));
247    }
248
249    #[test]
250    fn prose_sanitizer_strips_then_redacts() {
251        let out = sanitize_prose(
252            "I used Claude to write this.\nCo-Authored-By: X\nDone.",
253            &cfg(),
254        );
255        assert!(!out.to_lowercase().contains("claude"), "{out}");
256        assert!(out.contains("[REDACTED]"));
257        assert!(out.contains("Done."));
258    }
259
260    #[test]
261    fn redact_preserves_surrounding_bytes_with_multibyte_text() {
262        let tokens = vec!["claude".to_owned()];
263        let out = redact("日本語 CLAUDE で書いた 🤖", &tokens);
264        assert_eq!(out, "日本語 [REDACTED] で書いた 🤖");
265    }
266
267    #[test]
268    fn redact_terminates_when_replacement_contains_no_token() {
269        let tokens = vec!["a".to_owned()];
270        assert_eq!(redact("aaa", &tokens), "[REDACTED][REDACTED][REDACTED]");
271    }
272
273    #[test]
274    fn scan_counts_without_modifying() {
275        let leaks = scan(
276            "candidate B patch",
277            "Claude and claude and Gemini",
278            &cfg().vendor_tokens,
279        );
280        let claude = leaks.iter().find(|l| l.token == "claude").unwrap();
281        assert_eq!(claude.count, 2);
282        assert_eq!(claude.site, "candidate B patch");
283        assert!(leaks.iter().any(|l| l.token == "gemini"));
284    }
285
286    #[test]
287    fn warn_policy_leaves_the_patch_intact() {
288        let mut c = cfg();
289        c.on_leak = LeakPolicy::Warn;
290        let patch = "+// written by claude\n";
291        let (text, leaks) = sanitize_patch("candidate A patch", patch, &c);
292        assert_eq!(text, patch, "a warn must not rewrite the diff");
293        assert!(!leaks.is_empty());
294    }
295
296    #[test]
297    fn redact_policy_rewrites_the_patch() {
298        let mut c = cfg();
299        c.on_leak = LeakPolicy::Redact;
300        let (text, leaks) = sanitize_patch("candidate A patch", "+// by claude\n", &c);
301        assert!(text.contains("[REDACTED]"));
302        assert_eq!(leaks.len(), 1);
303    }
304
305    #[test]
306    fn hook_script_is_case_insensitive_sh() {
307        let script = commit_msg_hook(&cfg().strip_lines);
308        assert!(script.starts_with("#!/bin/sh\n"));
309        assert!(script.contains("[Cc][Oo]-[Aa][Uu][Tt][Hh][Oo][Rr][Ee][Dd]-[Bb][Yy]:"));
310        assert!(script.contains("mv \"$tmp\" \"$msg\""));
311    }
312
313    #[test]
314    fn sed_pattern_escapes_metacharacters() {
315        assert_eq!(sed_ci_pattern("a.b"), "[Aa]\\.[Bb]");
316        assert_eq!(sed_ci_pattern("x/y"), "[Xx]\\/[Yy]");
317        assert_eq!(sed_ci_pattern("\u{1f916}"), "\u{1f916}");
318    }
319}
magi/blind.rs

magi/
blind.rs
