snitch/
text.rs

1//! What is actually in a piece of text.
2//!
3//! A port of the web tool's `app/snitch/lib/text.ts`, kept behaviourally identical on purpose so
4//! the CLI, the MCP server and the website cannot disagree about what a piece of text contains.
5//! Change one, change the other, or the tool starts telling two stories.
6//!
7//! Everything reported here is CHECKABLE. Each finding names the exact code points it found, so a
8//! sceptical reader can verify it in any editor. Nothing is inferred from writing style: no "this
9//! looks like AI" guessing, because that is not measurable and would be the same class of false
10//! claim the image side has already been audited for.
11
12pub const ZWSP: char = '\u{200B}';
13pub const ZWNJ: char = '\u{200C}';
14pub const ZWJ: char = '\u{200D}';
15pub const BOM: char = '\u{FEFF}';
16pub const WORD_JOINER: char = '\u{2060}';
17pub const INVISIBLE_SEPARATOR: char = '\u{2063}';
18pub const FUNCTION_APPLICATION: char = '\u{2061}';
19
20/// ZWJ (U+200D) and ZWNJ (U+200C) are deliberately absent, and must stay absent. They are
21/// load-bearing: ZWJ builds emoji sequences such as a family or a profession, and both are
22/// standard orthography in Arabic, Persian, Kurdish and Indic scripts. Stripping them corrupts
23/// real writing in four languages and turns a family into three strangers. The web tool's own
24/// source records catching that in test. Only characters with no legitimate role in running text
25/// belong here.
26pub const SUSPICIOUS_ZERO_WIDTH: [char; 5] = [
27    ZWSP,
28    BOM,
29    WORD_JOINER,
30    INVISIBLE_SEPARATOR,
31    FUNCTION_APPLICATION,
32];
33
34pub const RLO: char = '\u{202E}';
35pub const LRO: char = '\u{202D}';
36
37pub const BIDI_CONTROLS: [char; 9] = [
38    '\u{202A}', // LRE
39    '\u{202B}', // RLE
40    '\u{202C}', // PDF, pop directional formatting
41    LRO, RLO, '\u{2066}', // LRI
42    '\u{2067}', // RLI
43    '\u{2068}', // FSI
44    '\u{2069}', // PDI
45];
46
47pub const ODD_SPACES: [(char, &str); 6] = [
48    ('\u{00A0}', "non-breaking space"),
49    ('\u{2007}', "figure space"),
50    ('\u{2009}', "thin space"),
51    ('\u{202F}', "narrow no-break space"),
52    ('\u{3000}', "ideographic space"),
53    ('\u{200A}', "hair space"),
54];
55
56pub const ORIGIN_TELLS: [(char, &str); 7] = [
57    ('\u{2018}', "curly opening quote"),
58    ('\u{2019}', "curly closing quote or apostrophe"),
59    ('\u{201C}', "curly opening double quote"),
60    ('\u{201D}', "curly closing double quote"),
61    ('\u{2026}', "ellipsis character"),
62    ('\u{2013}', "en dash"),
63    ('\u{2014}', "em dash"),
64];
65
66#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
67#[serde(rename_all = "lowercase")]
68pub enum Severity {
69    Alarming,
70    Notable,
71    Neutral,
72}
73
74#[derive(Debug, Clone, serde::Serialize)]
75pub struct Finding {
76    pub key: String,
77    pub label: String,
78    pub value: String,
79    pub severity: Severity,
80    pub note: String,
81    /// The exact characters found, for display and for the targeted strip.
82    pub chars: Vec<String>,
83}
84
85#[derive(Debug, Clone, serde::Serialize)]
86pub struct Cleaned {
87    pub text: String,
88    pub removed: usize,
89    pub removed_chars: Vec<String>,
90    pub findings: Vec<Finding>,
91}
92
93pub fn code_point(c: char) -> String {
94    format!("U+{:04X}", c as u32)
95}
96
97fn lookup(table: &[(char, &'static str)], c: char) -> Option<&'static str> {
98    table.iter().find(|(k, _)| *k == c).map(|(_, v)| *v)
99}
100
101/// Latin, Cyrillic or Greek, or None for anything else. Only these three confuse each other.
102fn script_of(c: char) -> Option<&'static str> {
103    let n = c as u32;
104    if (0x41..=0x5A).contains(&n) || (0x61..=0x7A).contains(&n) {
105        return Some("Latin");
106    }
107    if (0x0400..=0x04FF).contains(&n) {
108        return Some("Cyrillic");
109    }
110    if (0x0370..=0x03FF).contains(&n) {
111        return Some("Greek");
112    }
113    None
114}
115
116/// Words containing more than one alphabet.
117///
118/// This is the actual mechanism behind lookalike domains and impersonated names: a Cyrillic "а" is
119/// a different character from a Latin "a" and renders identically in most fonts.
120pub fn mixed_script_words(text: &str) -> Vec<(String, Vec<&'static str>)> {
121    let mut out = Vec::new();
122    for word in text.split_whitespace() {
123        if word.chars().count() < 2 {
124            continue;
125        }
126        let mut scripts: Vec<&'static str> = Vec::new();
127        for c in word.chars() {
128            if let Some(s) = script_of(c) {
129                if !scripts.contains(&s) {
130                    scripts.push(s);
131                }
132            }
133        }
134        if scripts.len() > 1 {
135            scripts.sort_unstable();
136            out.push((word.to_string(), scripts));
137        }
138    }
139    out
140}
141
142fn unique(chars: &[char]) -> Vec<char> {
143    let mut seen: Vec<char> = Vec::new();
144    for &c in chars {
145        if !seen.contains(&c) {
146            seen.push(c);
147        }
148    }
149    seen
150}
151
152fn joined(chars: &[char], f: impl Fn(char) -> String) -> String {
153    chars.iter().map(|&c| f(c)).collect::<Vec<_>>().join(", ")
154}
155
156/// Findings, in the same order and with the same keys as the web tool.
157pub fn analyse(text: &str) -> Vec<Finding> {
158    let mut findings: Vec<Finding> = Vec::new();
159    if text.is_empty() {
160        return findings;
161    }
162
163    let (mut hidden, mut bidi, mut spaces, mut tells) =
164        (Vec::new(), Vec::new(), Vec::new(), Vec::new());
165    for c in text.chars() {
166        if SUSPICIOUS_ZERO_WIDTH.contains(&c) {
167            hidden.push(c);
168        } else if BIDI_CONTROLS.contains(&c) {
169            bidi.push(c);
170        } else if lookup(&ODD_SPACES, c).is_some() {
171            spaces.push(c);
172        } else if lookup(&ORIGIN_TELLS, c).is_some() {
173            tells.push(c);
174        }
175    }
176
177    if !bidi.is_empty() {
178        let uniq = unique(&bidi);
179        let has_override = uniq.iter().any(|&c| c == RLO || c == LRO);
180        findings.push(Finding {
181            key: "bidi".into(),
182            label: "Direction-control characters".into(),
183            value: format!("{} found ({})", bidi.len(), joined(&uniq, code_point)),
184            severity: if has_override {
185                Severity::Alarming
186            } else {
187                Severity::Notable
188            },
189            note: if has_override {
190                "This text contains a right-to-left override. That character can make text display \
191                 in a different order than it is actually stored, which is the trick used to \
192                 disguise what a filename or a link really says."
193            } else {
194                "Direction-control characters are legitimate in mixed right-to-left and \
195                 left-to-right text, and are also used to disguise what text says. Worth knowing \
196                 they are here."
197            }
198            .into(),
199            chars: uniq.iter().map(|c| c.to_string()).collect(),
200        });
201    }
202
203    let mixed = mixed_script_words(text);
204    if !mixed.is_empty() {
205        findings.push(Finding {
206            key: "confusable".into(),
207            label: "Words mixing alphabets".into(),
208            value: mixed
209                .iter()
210                .take(4)
211                .map(|(word, scripts)| format!("{word} ({})", scripts.join(" + ")))
212                .collect::<Vec<_>>()
213                .join(", "),
214            severity: Severity::Alarming,
215            note: "These words contain letters from more than one alphabet. A Cyrillic a and a \
216                   Latin a look identical in almost every font, which is how lookalike domains \
217                   and impersonated names work. If you did not write this in more than one \
218                   language, treat it as deliberate."
219                .into(),
220            chars: Vec::new(),
221        });
222    }
223
224    if !hidden.is_empty() {
225        let uniq = unique(&hidden);
226        findings.push(Finding {
227            key: "hidden".into(),
228            label: "Invisible characters".into(),
229            value: format!("{} found ({})", hidden.len(), joined(&uniq, code_point)),
230            severity: Severity::Notable,
231            note:
232                "Invisible characters. They survive copying and pasting, they are not visible in \
233                   any editor, and they can be used to fingerprint a copy of a document. They are \
234                   also occasionally harmless leftovers from a web page."
235                    .into(),
236            chars: uniq.iter().map(|c| c.to_string()).collect(),
237        });
238    }
239
240    if !spaces.is_empty() {
241        let uniq = unique(&spaces);
242        findings.push(Finding {
243            key: "space".into(),
244            label: "Unusual spaces".into(),
245            value: format!(
246                "{} found ({})",
247                spaces.len(),
248                joined(&uniq, |c| lookup(&ODD_SPACES, c)
249                    .unwrap_or("space")
250                    .to_string())
251            ),
252            severity: Severity::Neutral,
253            note: "Spaces that are not the ordinary space character. They survive copying, they \
254                   break search and find, and they usually arrive from a web page or a word \
255                   processor rather than being typed."
256                .into(),
257            chars: uniq.iter().map(|c| c.to_string()).collect(),
258        });
259    }
260
261    if !tells.is_empty() {
262        let uniq = unique(&tells);
263        let plural = if tells.len() == 1 { "" } else { "s" };
264        findings.push(Finding {
265            key: "origin".into(),
266            label: "Came through a word processor".into(),
267            value: format!(
268                "{} substituted character{plural} ({})",
269                tells.len(),
270                joined(&uniq, |c| lookup(&ORIGIN_TELLS, c)
271                    .unwrap_or("mark")
272                    .to_string())
273            ),
274            severity: Severity::Neutral,
275            note:
276                "Curly quotes, ellipsis characters and long dashes are inserted automatically by \
277                   Word, Google Docs and most publishing tools. Plain typing in a code editor does \
278                   not produce them. This is a weak hint about where the text came from, not proof \
279                   of anything."
280                    .into(),
281            chars: uniq.iter().map(|c| c.to_string()).collect(),
282        });
283    }
284
285    if findings.is_empty() {
286        findings.push(Finding {
287            key: "clean".into(),
288            label: "Nothing hidden".into(),
289            value: "Plain text".into(),
290            severity: Severity::Neutral,
291            note: "No invisible characters, no direction controls, no mixed alphabets and no \
292                   word-processor substitutions. What you see is what is stored."
293                .into(),
294            chars: Vec::new(),
295        });
296    }
297
298    findings
299}
300
301/// Remove exactly the characters given, and nothing else.
302pub fn strip_chars(text: &str, unwanted: &[char]) -> String {
303    if unwanted.is_empty() {
304        return text.to_string();
305    }
306    text.chars().filter(|c| !unwanted.contains(c)).collect()
307}
308
309/// Remove the tracking characters and leave the writing alone.
310///
311/// Removes the invisible characters and the direction controls, because neither has a role in
312/// ordinary running text. Leaves odd spaces and word-processor substitutions in place: they are
313/// reported so a reader knows they are there, and removing them silently would edit someone's
314/// prose. Never touches ZWJ or ZWNJ, which real writing depends on.
315pub fn clean(text: &str) -> Cleaned {
316    let findings = analyse(text);
317    let removable: Vec<char> = findings
318        .iter()
319        .filter(|f| f.key == "hidden" || f.key == "bidi")
320        .flat_map(|f| f.chars.iter().filter_map(|s| s.chars().next()))
321        .collect();
322    let cleaned = strip_chars(text, &removable);
323    Cleaned {
324        removed: text.chars().count() - cleaned.chars().count(),
325        removed_chars: removable.iter().map(|&c| code_point(c)).collect(),
326        text: cleaned,
327        findings,
328    }
329}
snitch/text.rs

snitch/
text.rs
