magi/
github_text.rs

1//! Posting gate shared by GitHub titles, descriptions and comments.
2use crate::run::RunState;
3use crate::scrub::{Identity, scrub};
4
5/// Fixed fallback for a rejected title.
6pub const NEUTRAL_TITLE: &str = "chore: update repository";
7/// Fixed fallback for a rejected description or comment.
8pub const NEUTRAL_BODY: &str = "Generated GitHub text was withheld by the magi posting gate. Review the branch diff and the local run report for details.";
9
10/// Categories only: diagnostics must never repeat the offending secret.
11#[derive(Debug, Clone, Copy, PartialEq, Eq)]
12pub enum Violation {
13    /// Title contains a meaningful share of non-English letters.
14    TitleLanguage,
15    /// Unquoted body prose contains a meaningful share of non-English letters.
16    BodyLanguage,
17    /// Shared redaction rules found sensitive or local data.
18    SensitiveData,
19}
20
21/// Pure checker. Language exemptions never exempt sensitive data.
22pub fn check(title: &str, body: &str) -> Vec<Violation> {
23    let mut out = Vec::new();
24    if non_english(title) {
25        out.push(Violation::TitleLanguage);
26    }
27    if non_english(&prose(body)) {
28        out.push(Violation::BodyLanguage);
29    }
30    let id = Identity::default();
31    if scrub(title, &id) != title || scrub(body, &id) != body {
32        out.push(Violation::SensitiveData);
33    }
34    out
35}
36
37/// Function words and common verbs of the Latin-script languages most likely
38/// to appear, chosen to avoid ordinary English words.
39const FOREIGN_WORDS: &[&str] = &[
40    "este",
41    "esta",
42    "para",
43    "los",
44    "las",
45    "del",
46    "una",
47    "que",
48    "por",
49    "errores",
50    "corregir",
51    "agrega",
52    "cambio",
53    "solicitudes",
54    "fallidas",
55    "reintentos",
56    "les",
57    "des",
58    "pour",
59    "avec",
60    "dans",
61    "est",
62    "une",
63    "pas",
64    "und",
65    "der",
66    "das",
67    "nicht",
68    "mit",
69    "ein",
70    "eine",
71    "für",
72    "wird",
73    "não",
74    "uma",
75    "della",
76    "che",
77    "con",
78    "fehler",
79    "corrigir",
80    "erreurs",
81];
82
83fn foreign_words(text: &str) -> bool {
84    let words: Vec<String> = text
85        .split(|c: char| !c.is_alphabetic())
86        .filter(|w| !w.is_empty())
87        .map(str::to_lowercase)
88        .collect();
89    let hits = words
90        .iter()
91        .filter(|w| FOREIGN_WORDS.contains(&w.as_str()))
92        .count();
93    hits >= 2 && hits * 4 >= words.len()
94}
95
96fn non_english(text: &str) -> bool {
97    if foreign_words(text) {
98        return true;
99    }
100    let letters = text.chars().filter(|c| c.is_alphabetic()).count();
101    let foreign = text
102        .chars()
103        .filter(|c| c.is_alphabetic() && !c.is_ascii())
104        .count();
105    // Accents and isolated identifiers are common in otherwise English prose.
106    (foreign >= 4 && foreign * 10 >= letters.max(1))
107        || text.lines().any(|line| {
108            let letters = line.chars().filter(|c| c.is_alphabetic()).count();
109            let foreign = line
110                .chars()
111                .filter(|c| c.is_alphabetic() && !c.is_ascii())
112                .count();
113            foreign >= 4 && foreign * 2 >= letters.max(1)
114        })
115}
116
117fn prose(body: &str) -> String {
118    let mut out = String::new();
119    let mut details = 0usize;
120    let mut quote = false;
121    let mut fence: Option<(char, usize)> = None;
122    let mut inline = 0usize;
123    for line in body.lines() {
124        let trimmed = line.trim_start();
125        if let Some((marker, width)) = fence {
126            if trimmed.chars().take_while(|c| *c == marker).count() >= width {
127                fence = None;
128            }
129            continue;
130        }
131        let marker = trimmed.chars().next().unwrap_or(' ');
132        let width = trimmed.chars().take_while(|c| *c == marker).count();
133        if matches!(marker, '`' | '~') && width >= 3 {
134            fence = Some((marker, width));
135            continue;
136        }
137        if trimmed.starts_with('>') || line.starts_with("    ") || line.starts_with('\t') {
138            continue;
139        }
140        let mut rest = line;
141        while !rest.is_empty() {
142            if rest.starts_with('<')
143                && let Some(end) = rest.find('>')
144            {
145                let tag = rest[..=end].to_ascii_lowercase();
146                if tag.starts_with("<details") {
147                    details += 1;
148                } else if tag == "</details>" {
149                    details = details.saturating_sub(1);
150                } else if tag.starts_with("<blockquote") {
151                    quote = true;
152                } else if tag == "</blockquote>" {
153                    quote = false;
154                }
155                rest = &rest[end + 1..];
156                continue;
157            }
158            if rest.starts_with('`') {
159                let n = rest.chars().take_while(|c| *c == '`').count();
160                if inline == 0 {
161                    inline = n;
162                } else if inline == n {
163                    inline = 0;
164                }
165                rest = &rest[n..];
166                continue;
167            }
168            let c = rest.chars().next().expect("nonempty");
169            if details == 0 && !quote && inline == 0 {
170                out.push(c);
171            }
172            rest = &rest[c.len_utf8()..];
173        }
174        out.push('\n');
175    }
176    out
177}
178
179/// Scrub local identity and pattern matches, then replace failing prose.
180/// Every intervention is recorded without copying the offending material.
181pub fn prepare(state: &mut RunState, title: &str, body: &str) -> (String, String) {
182    let id = Identity::current();
183    let clean_title = scrub(title, &id);
184    let clean_body = scrub(body, &id);
185    let violations = check(title, body);
186    if clean_title != title || clean_body != body {
187        state.event("github-text", "sensitive data removed before posting");
188    }
189    let language = state.config.graph.github_text_guard;
190    let title = if language && violations.contains(&Violation::TitleLanguage) {
191        state.event("github-text", "title replaced with neutral English text");
192        NEUTRAL_TITLE.to_owned()
193    } else {
194        clean_title
195    };
196    let body = if language && violations.contains(&Violation::BodyLanguage) {
197        state.event("github-text", "body replaced with neutral English text");
198        NEUTRAL_BODY.to_owned()
199    } else {
200        clean_body
201    };
202    (title, body)
203}
204
205#[cfg(test)]
206mod tests {
207    use super::*;
208
209    #[test]
210    fn github_text_language_and_exemptions() {
211        assert_eq!(
212            check("fix: retries", "日本語で変更の説明を書きます。"),
213            vec![Violation::BodyLanguage]
214        );
215        assert!(check("fix: retries", "Add retries for failed requests.\n<details>\n<summary>Original task</summary>\n日本語の元の依頼です。\n</details>").is_empty());
216        for body in [
217            "Add retries. `日本語の識別子`",
218            "Add retries.\n```text\n日本語のコードです\n```",
219            "Add retries.\n> 日本語の引用です",
220            "Add retries.\n<blockquote>日本語の引用です</blockquote>",
221            "Update café names.",
222            "Change src/graph.rs and tests/common/mod.rs.",
223        ] {
224            assert!(check("fix: retries", body).is_empty(), "{body}");
225        }
226    }
227
228    #[test]
229    fn github_text_sensitive_data_in_all_sections() {
230        for secret in [
231            "/Users/example/repo",
232            "/home/example/repo",
233            "C:\\Users\\Example\\repo",
234            "/private/tmp/work",
235            "dev@example.test",
236            "ghp_abcdefghijklmnopqrstuv",
237            "github_pat_abcdefghijklmnopqrstuv",
238            "sk-abcdefghijklmnopqrstuv",
239            "AKIAABCDEFGHIJKLMNOP",
240            "password=example",
241            "password=\"example\"",
242            "token aBcdEfgHijkLmn0123456789",
243            "buildbox.local",
244            "token=abcdefghijklmnop012345",
245            "Authorization: Bearer abcdefghijklmnop",
246            "hostname=buildbox",
247            "username=example",
248            "10.2.3.4",
249            "fe80::1",
250        ] {
251            assert!(
252                check(secret, "").contains(&Violation::SensitiveData),
253                "{secret}"
254            );
255            assert!(
256                check(
257                    "fix: retries",
258                    &format!("<details>\n`{secret}`\n</details>")
259                )
260                .contains(&Violation::SensitiveData),
261                "{secret}"
262            );
263        }
264        for clean in [
265            "https://github.com/example/repo",
266            "src/graph.rs",
267            "docs/home/example",
268            "/api/v1/runs",
269            "v1.2.3",
270            "std::io::Error",
271            "Use the token from the environment.",
272        ] {
273            assert!(check("fix: retries", clean).is_empty(), "{clean}");
274        }
275    }
276
277    #[test]
278    fn github_text_config_only_disables_language_and_records_interventions() {
279        let mut state = RunState::new(
280            ".".into(),
281            "main".into(),
282            "abc".into(),
283            "task".into(),
284            crate::config::Config::default(),
285        );
286        let (_, body) = prepare(
287            &mut state,
288            "fix: retries",
289            "日本語の説明を書きます。 token=secret",
290        );
291        assert_eq!(body, NEUTRAL_BODY);
292        assert!(!state.events.is_empty());
293        state.config.graph.github_text_guard = false;
294        let (_, body) = prepare(
295            &mut state,
296            "fix: retries",
297            "日本語の説明を書きます。 token=secret",
298        );
299        assert!(body.contains("日本語"));
300        assert!(!body.contains("secret"));
301        assert!(
302            check("fix: retries", &body)
303                .iter()
304                .all(|v| *v != Violation::SensitiveData)
305        );
306    }
307
308    #[test]
309    fn github_text_fixed_fallback_passes() {
310        assert!(check(NEUTRAL_TITLE, NEUTRAL_BODY).is_empty());
311    }
312}
313
314#[cfg(test)]
315mod review_round_tests {
316    use super::*;
317
318    #[test]
319    fn latin_script_non_english_is_flagged() {
320        assert!(check("Corregir errores", "fix: retry").contains(&Violation::TitleLanguage));
321        assert!(
322            check(
323                "fix: retries",
324                "Este cambio agrega reintentos para solicitudes fallidas."
325            )
326            .contains(&Violation::BodyLanguage)
327        );
328        assert!(
329            check(
330                "fix: retry failed requests",
331                "Adds retries for failed requests."
332            )
333            .is_empty()
334        );
335    }
336
337    #[test]
338    fn quoted_json_credentials_are_sensitive() {
339        let body = "Example: {\"password\": \"hunter2\"}";
340        assert!(check("t", body).contains(&Violation::SensitiveData));
341        assert!(!crate::scrub::scrub(body, &Identity::default()).contains("hunter2"));
342    }
343}
344
345#[cfg(test)]
346mod quoted_value_tests {
347    use crate::scrub::{Identity, scrub};
348
349    #[test]
350    fn quoted_values_are_redacted_whole() {
351        for v in ["correct horse battery staple", ",hunter2"] {
352            let out = scrub(
353                &format!("{{\"password\": \"{v}\"}} ok"),
354                &Identity::default(),
355            );
356            assert!(!out.contains("horse") && !out.contains("hunter2"), "{out}");
357            assert!(out.ends_with("ok"), "{out}");
358        }
359    }
360}
magi/github_text.rs

magi/
github_text.rs
