magi/
github_text.rs

1//! Posting gate shared by GitHub titles, descriptions and comments.
2use crate::run::RunState;
3use crate::scrub::{Identity, scrub};
4
5/// Fixed fallback for a rejected title.
6pub const NEUTRAL_TITLE: &str = "chore: update repository";
7/// Fixed fallback for a rejected description or comment.
8pub const NEUTRAL_BODY: &str = "Generated GitHub text was withheld by the magi posting gate. Review the branch diff and the local run report for details.";
9
10/// Categories only: diagnostics must never repeat the offending secret.
11#[derive(Debug, Clone, Copy, PartialEq, Eq)]
12pub enum Violation {
13    /// Title contains a meaningful share of non-English letters.
14    TitleLanguage,
15    /// Unquoted body prose contains a meaningful share of non-English letters.
16    BodyLanguage,
17    /// Shared redaction rules found sensitive or local data.
18    SensitiveData,
19}
20
21/// Pure checker. Language exemptions never exempt sensitive data.
22pub fn check(title: &str, body: &str) -> Vec<Violation> {
23    let mut out = Vec::new();
24    if non_english(title) {
25        out.push(Violation::TitleLanguage);
26    }
27    if non_english(&prose(body)) {
28        out.push(Violation::BodyLanguage);
29    }
30    let id = Identity::default();
31    if scrub(title, &id) != title || scrub(body, &id) != body {
32        out.push(Violation::SensitiveData);
33    }
34    out
35}
36
37/// Function words and common verbs of the Latin-script languages most likely
38/// to appear, chosen to avoid ordinary English words.
39const FOREIGN_WORDS: &[&str] = &[
40    "este",
41    "esta",
42    "para",
43    "los",
44    "las",
45    "del",
46    "una",
47    "que",
48    "por",
49    "errores",
50    "corregir",
51    "agrega",
52    "cambio",
53    "solicitudes",
54    "fallidas",
55    "reintentos",
56    "les",
57    "des",
58    "pour",
59    "avec",
60    "dans",
61    "est",
62    "une",
63    "pas",
64    "und",
65    "der",
66    "das",
67    "nicht",
68    "mit",
69    "ein",
70    "eine",
71    "für",
72    "wird",
73    "não",
74    "uma",
75    "della",
76    "che",
77    "con",
78    "fehler",
79    "corrigir",
80    "erreurs",
81];
82
83fn foreign_words(text: &str) -> bool {
84    let words: Vec<String> = text
85        .split(|c: char| !c.is_alphabetic())
86        .filter(|w| !w.is_empty())
87        .map(str::to_lowercase)
88        .collect();
89    let hits = words
90        .iter()
91        .filter(|w| FOREIGN_WORDS.contains(&w.as_str()))
92        .count();
93    hits >= 2 && hits * 4 >= words.len()
94}
95
96fn non_english(text: &str) -> bool {
97    if foreign_words(text) {
98        return true;
99    }
100    let letters = text.chars().filter(|c| c.is_alphabetic()).count();
101    let foreign = text
102        .chars()
103        .filter(|c| c.is_alphabetic() && !c.is_ascii())
104        .count();
105    // Accents and isolated identifiers are common in otherwise English prose.
106    (foreign >= 4 && foreign * 10 >= letters.max(1))
107        || text.lines().any(|line| {
108            let letters = line.chars().filter(|c| c.is_alphabetic()).count();
109            let foreign = line
110                .chars()
111                .filter(|c| c.is_alphabetic() && !c.is_ascii())
112                .count();
113            foreign >= 4 && foreign * 2 >= letters.max(1)
114        })
115}
116
117/// Offset just past the first backtick run of exactly `n` in `text`, if any.
118/// An unmatched opener is literal text in Markdown, so it exempts nothing.
119fn closing_run(text: &str, n: usize) -> Option<usize> {
120    let mut at = 0;
121    while let Some(i) = text[at..].find('`') {
122        let start = at + i;
123        let len = text[start..].chars().take_while(|c| *c == '`').count();
124        if len == n {
125            return Some(start + len);
126        }
127        at = start + len;
128    }
129    None
130}
131
132fn prose(body: &str) -> String {
133    let mut out = String::new();
134    let mut details = 0usize;
135    let mut quote = false;
136    let mut fence: Option<(char, usize)> = None;
137    for line in body.lines() {
138        let trimmed = line.trim_start();
139        if let Some((marker, width)) = fence {
140            if trimmed.chars().take_while(|c| *c == marker).count() >= width {
141                fence = None;
142            }
143            continue;
144        }
145        let marker = trimmed.chars().next().unwrap_or(' ');
146        let width = trimmed.chars().take_while(|c| *c == marker).count();
147        if matches!(marker, '`' | '~') && width >= 3 {
148            fence = Some((marker, width));
149            continue;
150        }
151        if trimmed.starts_with('>') || line.starts_with("    ") || line.starts_with('\t') {
152            continue;
153        }
154        let mut rest = line;
155        while !rest.is_empty() {
156            if rest.starts_with('<')
157                && let Some(end) = rest.find('>')
158            {
159                let tag = rest[..=end].to_ascii_lowercase();
160                if tag.starts_with("<details") {
161                    details += 1;
162                } else if tag == "</details>" {
163                    details = details.saturating_sub(1);
164                } else if tag.starts_with("<blockquote") {
165                    quote = true;
166                } else if tag == "</blockquote>" {
167                    quote = false;
168                }
169                rest = &rest[end + 1..];
170                continue;
171            }
172            if rest.starts_with('`') {
173                let n = rest.chars().take_while(|c| *c == '`').count();
174                rest = match closing_run(&rest[n..], n) {
175                    Some(end) => &rest[n + end..],
176                    None => &rest[n..],
177                };
178                continue;
179            }
180            let c = rest.chars().next().expect("nonempty");
181            if details == 0 && !quote {
182                out.push(c);
183            }
184            rest = &rest[c.len_utf8()..];
185        }
186        out.push('\n');
187    }
188    out
189}
190
191/// Scrub local identity and pattern matches, then replace failing prose.
192/// Every intervention is recorded without copying the offending material.
193pub fn prepare(state: &mut RunState, title: &str, body: &str) -> (String, String) {
194    let id = Identity::current();
195    let clean_title = scrub(title, &id);
196    let clean_body = scrub(body, &id);
197    let violations = check(title, body);
198    if clean_title != title || clean_body != body {
199        state.event("github-text", "sensitive data removed before posting");
200    }
201    let language = state.config.graph.github_text_guard;
202    let title = if language && violations.contains(&Violation::TitleLanguage) {
203        state.event("github-text", "title replaced with neutral English text");
204        NEUTRAL_TITLE.to_owned()
205    } else {
206        clean_title
207    };
208    let body = if language && violations.contains(&Violation::BodyLanguage) {
209        state.event("github-text", "body replaced with neutral English text");
210        NEUTRAL_BODY.to_owned()
211    } else {
212        clean_body
213    };
214    (title, body)
215}
216
217#[cfg(test)]
218mod tests {
219    use super::*;
220
221    #[test]
222    fn github_text_language_and_exemptions() {
223        assert_eq!(
224            check("fix: retries", "日本語で変更の説明を書きます。"),
225            vec![Violation::BodyLanguage]
226        );
227        assert!(check("fix: retries", "Add retries for failed requests.\n<details>\n<summary>Original task</summary>\n日本語の元の依頼です。\n</details>").is_empty());
228        for body in [
229            "Fix `cache\n\n日本語の説明を書きます。",
230            "Fix `cache 日本語の説明を書きます。",
231        ] {
232            assert_eq!(
233                check("fix: retries", body),
234                vec![Violation::BodyLanguage],
235                "{body}"
236            );
237        }
238        for body in [
239            "Add retries. `日本語の識別子`",
240            "Add retries.\n```text\n日本語のコードです\n```",
241            "Add retries.\n> 日本語の引用です",
242            "Add retries.\n<blockquote>日本語の引用です</blockquote>",
243            "Add retries. ``日本語 ` の識別子``",
244            "Update café names.",
245            "Change src/graph.rs and tests/common/mod.rs.",
246        ] {
247            assert!(check("fix: retries", body).is_empty(), "{body}");
248        }
249    }
250
251    #[test]
252    fn github_text_sensitive_data_in_all_sections() {
253        for secret in [
254            "/Users/example/repo",
255            "/home/example/repo",
256            "C:\\Users\\Example\\repo",
257            "/private/tmp/work",
258            "dev@example.test",
259            "ghp_abcdefghijklmnopqrstuv",
260            "github_pat_abcdefghijklmnopqrstuv",
261            "sk-abcdefghijklmnopqrstuv",
262            "AKIAABCDEFGHIJKLMNOP",
263            "password=example",
264            "password=\"example\"",
265            "token aBcdEfgHijkLmn0123456789",
266            "buildbox.local",
267            "token=abcdefghijklmnop012345",
268            "Authorization: Bearer abcdefghijklmnop",
269            "hostname=buildbox",
270            "username=example",
271            "10.2.3.4",
272            "fe80::1",
273        ] {
274            assert!(
275                check(secret, "").contains(&Violation::SensitiveData),
276                "{secret}"
277            );
278            assert!(
279                check(
280                    "fix: retries",
281                    &format!("<details>\n`{secret}`\n</details>")
282                )
283                .contains(&Violation::SensitiveData),
284                "{secret}"
285            );
286        }
287        for clean in [
288            "https://github.com/example/repo",
289            "src/graph.rs",
290            "docs/home/example",
291            "/api/v1/runs",
292            "v1.2.3",
293            "std::io::Error",
294            "Use the token from the environment.",
295        ] {
296            assert!(check("fix: retries", clean).is_empty(), "{clean}");
297        }
298    }
299
300    #[test]
301    fn github_text_config_only_disables_language_and_records_interventions() {
302        let mut state = RunState::new(
303            ".".into(),
304            "main".into(),
305            "abc".into(),
306            "task".into(),
307            crate::config::Config::default(),
308        );
309        let (_, body) = prepare(
310            &mut state,
311            "fix: retries",
312            "日本語の説明を書きます。 token=secret",
313        );
314        assert_eq!(body, NEUTRAL_BODY);
315        assert!(!state.events.is_empty());
316        state.config.graph.github_text_guard = false;
317        let (_, body) = prepare(
318            &mut state,
319            "fix: retries",
320            "日本語の説明を書きます。 token=secret",
321        );
322        assert!(body.contains("日本語"));
323        assert!(!body.contains("secret"));
324        assert!(
325            check("fix: retries", &body)
326                .iter()
327                .all(|v| *v != Violation::SensitiveData)
328        );
329    }
330
331    #[test]
332    fn github_text_fixed_fallback_passes() {
333        assert!(check(NEUTRAL_TITLE, NEUTRAL_BODY).is_empty());
334    }
335}
336
337#[cfg(test)]
338mod review_round_tests {
339    use super::*;
340
341    #[test]
342    fn latin_script_non_english_is_flagged() {
343        assert!(check("Corregir errores", "fix: retry").contains(&Violation::TitleLanguage));
344        assert!(
345            check(
346                "fix: retries",
347                "Este cambio agrega reintentos para solicitudes fallidas."
348            )
349            .contains(&Violation::BodyLanguage)
350        );
351        assert!(
352            check(
353                "fix: retry failed requests",
354                "Adds retries for failed requests."
355            )
356            .is_empty()
357        );
358    }
359
360    #[test]
361    fn quoted_json_credentials_are_sensitive() {
362        let body = "Example: {\"password\": \"hunter2\"}";
363        assert!(check("t", body).contains(&Violation::SensitiveData));
364        assert!(!crate::scrub::scrub(body, &Identity::default()).contains("hunter2"));
365    }
366}
367
368#[cfg(test)]
369mod quoted_value_tests {
370    use crate::scrub::{Identity, scrub};
371
372    #[test]
373    fn quoted_values_are_redacted_whole() {
374        for v in ["correct horse battery staple", ",hunter2"] {
375            let out = scrub(
376                &format!("{{\"password\": \"{v}\"}} ok"),
377                &Identity::default(),
378            );
379            assert!(!out.contains("horse") && !out.contains("hunter2"), "{out}");
380            assert!(out.ends_with("ok"), "{out}");
381        }
382    }
383}
magi/github_text.rs

magi/
github_text.rs
