aitextwatermarkremover_tools/
lib.rs

1//! Local utilities for scanning and removing invisible Unicode characters.
2//! Independent third-party helper inspired by https://aitextwatermarkremover.com/
3
4use regex::Regex;
5use std::collections::HashMap;
6use std::sync::OnceLock;
7
8#[derive(Debug, Clone, PartialEq, Eq)]
9pub struct Finding {
10    pub index: usize,
11    pub character: char,
12    pub code_point: String,
13    pub name: String,
14}
15
16fn names() -> &'static HashMap<char, &'static str> {
17    static NAMES: OnceLock<HashMap<char, &'static str>> = OnceLock::new();
18    NAMES.get_or_init(|| {
19        HashMap::from([
20            ('\u{00AD}', "SOFT HYPHEN"),
21            ('\u{034F}', "COMBINING GRAPHEME JOINER"),
22            ('\u{061C}', "ARABIC LETTER MARK"),
23            ('\u{115F}', "HANGUL CHOSEONG FILLER"),
24            ('\u{1160}', "HANGUL JUNGSEONG FILLER"),
25            ('\u{17B4}', "KHMER VOWEL INHERENT AQ"),
26            ('\u{17B5}', "KHMER VOWEL INHERENT AA"),
27            ('\u{180B}', "MONGOLIAN FREE VARIATION SELECTOR ONE"),
28            ('\u{180C}', "MONGOLIAN FREE VARIATION SELECTOR TWO"),
29            ('\u{180D}', "MONGOLIAN FREE VARIATION SELECTOR THREE"),
30            ('\u{180F}', "MONGOLIAN FREE VARIATION SELECTOR FOUR"),
31            ('\u{200B}', "ZERO WIDTH SPACE"),
32            ('\u{200C}', "ZERO WIDTH NON-JOINER"),
33            ('\u{200D}', "ZERO WIDTH JOINER"),
34            ('\u{200E}', "LEFT-TO-RIGHT MARK"),
35            ('\u{200F}', "RIGHT-TO-LEFT MARK"),
36            ('\u{202A}', "LEFT-TO-RIGHT EMBEDDING"),
37            ('\u{202B}', "RIGHT-TO-LEFT EMBEDDING"),
38            ('\u{202C}', "POP DIRECTIONAL FORMATTING"),
39            ('\u{202D}', "LEFT-TO-RIGHT OVERRIDE"),
40            ('\u{202E}', "RIGHT-TO-LEFT OVERRIDE"),
41            ('\u{202F}', "NARROW NO-BREAK SPACE"),
42            ('\u{205F}', "MEDIUM MATHEMATICAL SPACE"),
43            ('\u{2060}', "WORD JOINER"),
44            ('\u{2061}', "FUNCTION APPLICATION"),
45            ('\u{2062}', "INVISIBLE TIMES"),
46            ('\u{2063}', "INVISIBLE SEPARATOR"),
47            ('\u{2064}', "INVISIBLE PLUS"),
48            ('\u{2066}', "LEFT-TO-RIGHT ISOLATE"),
49            ('\u{2067}', "RIGHT-TO-LEFT ISOLATE"),
50            ('\u{2068}', "FIRST STRONG ISOLATE"),
51            ('\u{2069}', "POP DIRECTIONAL ISOLATE"),
52            ('\u{206A}', "INHIBIT SYMMETRIC SWAPPING"),
53            ('\u{206B}', "ACTIVATE SYMMETRIC SWAPPING"),
54            ('\u{206C}', "INHIBIT ARABIC FORM SHAPING"),
55            ('\u{206D}', "ACTIVATE ARABIC FORM SHAPING"),
56            ('\u{206E}', "NATIONAL DIGIT SHAPES"),
57            ('\u{206F}', "NOMINAL DIGIT SHAPES"),
58            ('\u{FE00}', "VARIATION SELECTOR-1"),
59            ('\u{FE01}', "VARIATION SELECTOR-2"),
60            ('\u{FE02}', "VARIATION SELECTOR-3"),
61            ('\u{FE03}', "VARIATION SELECTOR-4"),
62            ('\u{FE0E}', "TEXT VARIATION SELECTOR-15"),
63            ('\u{FE0F}', "EMOJI VARIATION SELECTOR-16"),
64            ('\u{FEFF}', "ZERO WIDTH NO-BREAK SPACE"),
65        ])
66    })
67}
68
69pub fn scan_invisible_characters(text: &str) -> Vec<Finding> {
70    let mut findings = Vec::new();
71    for (index, character) in text.char_indices() {
72        if let Some(name) = names().get(&character) {
73            findings.push(Finding {
74                index,
75                character,
76                code_point: format!("U+{:04X}", character as u32),
77                name: (*name).to_string(),
78            });
79        }
80    }
81    findings
82}
83
84pub fn remove_invisible_characters(text: &str) -> String {
85    text.chars()
86        .filter(|character| !names().contains_key(character))
87        .collect()
88}
89
90pub fn strip_markdown_paste_residue(text: &str) -> String {
91    static HEADING: OnceLock<Regex> = OnceLock::new();
92    static ESCAPED: OnceLock<Regex> = OnceLock::new();
93    static WRAP: OnceLock<Regex> = OnceLock::new();
94    static TRAIL: OnceLock<Regex> = OnceLock::new();
95    let text = HEADING
96        .get_or_init(|| Regex::new(r"(?m)^\s{0,3}(?:#{1,6}|[-*+] |\d+\. |>)\s?").unwrap())
97        .replace_all(text, "");
98    let text = ESCAPED
99        .get_or_init(|| Regex::new(r"\\([\\`*_{}\[\]()#+.!|>~-])").unwrap())
100        .replace_all(&text, "$1");
101    let text = WRAP
102        .get_or_init(|| Regex::new(r"(\*\*|__|~~|`)").unwrap())
103        .replace_all(&text, "");
104    let text = TRAIL
105        .get_or_init(|| Regex::new(r"[ \t]+\n").unwrap())
106        .replace_all(&text, "\n");
107    let mut out = text.into_owned();
108    while out.contains("\n\n\n") {
109        out = out.replace("\n\n\n", "\n\n");
110    }
111    out
112}
113
114#[cfg(test)]
115mod tests {
116    use super::*;
117
118    #[test]
119    fn scan_reports_codepoints_and_positions() {
120        let findings = scan_invisible_characters("A\u{200B}\u{202F}B");
121        assert_eq!(findings[0].code_point, "U+200B");
122        assert_eq!(findings[0].index, 1);
123        assert_eq!(findings[1].code_point, "U+202F");
124    }
125
126    #[test]
127    fn remove_preserves_visible_text() {
128        assert_eq!(remove_invisible_characters("A\u{200B} B\u{FEFF}"), "A B");
129    }
130
131    #[test]
132    fn tidy_markdown() {
133        assert_eq!(
134            strip_markdown_paste_residue("## Heading\n- **Item**\n\\[link\\]"),
135            "Heading\nItem\n[link]"
136        );
137    }
138}
aitextwatermarkremover_tools/lib.rs

aitextwatermarkremover_tools/
lib.rs
