eunicode/
unicode_string.rs

1use crate::convert::{
2    char_identifier_to_string, general_category_to_string, unicode_block_to_string,
3    unicode_script_to_string,
4};
5use ammonia::Builder;
6use charname::get_name;
7use deunicode::deunicode;
8use limace::Slugifier;
9use linkify::LinkFinder;
10use prettytable::{Cell as TableCell, Row, Table, format::consts::FORMAT_CLEAN, row};
11use rustrict::CensorStr;
12use std::{collections::HashSet, marker::PhantomData, process::exit};
13use ucd::Codepoint;
14use unicode_security::{
15    GeneralSecurityProfile, RestrictionLevel::ASCIIOnly, RestrictionLevelDetection, skeleton,
16};
17
18/// TypeState state definitions
19pub mod string_states {
20    use super::StringState;
21
22    pub struct RawInput {}
23    pub struct CleanedText {}
24
25    /// RawInput state: may data may contain dangerous characters - allows detect and chars operations to examine content
26    impl StringState for RawInput {}
27
28    /// CleanedText state: dangerous characters have been removed - allows strip, defang, censor, and sluggify operations
29    impl StringState for CleanedText {}
30}
31
32// TODO do we want to seal this trait? probably
33pub trait StringState {}
34
35/// TypeState wrapper for text processing
36pub struct UnicodeString<S: StringState> {
37    text: String,
38    _marker: PhantomData<S>,
39}
40
41impl UnicodeString<string_states::RawInput> {
42    pub fn new(text: String) -> Self {
43        Self {
44            text,
45            _marker: PhantomData,
46        }
47    }
48
49    pub fn to_string(&self) -> &str {
50        &self.text
51    }
52
53    /// Normalize Unicode characters to only safe, ASCII text chars and transition to CleanedText state
54    pub fn clean(self) -> UnicodeString<string_states::CleanedText> {
55        UnicodeString::<string_states::CleanedText> {
56            text: deunicode(&self.text),
57            _marker: PhantomData,
58        }
59    }
60
61    fn character_info(index: &usize, c: char) -> Row {
62        Row::new(vec![
63            TableCell::new(&index.to_string()),
64            // NOTE: we're calling deunicode here to avoid printing potentially dangerous characters
65            // TODO might want to print all "safe" printable characters to allow better analysis
66            TableCell::new(&deunicode(&c.to_string())),
67            TableCell::new(general_category_to_string(c.category())),
68            TableCell::new(unicode_block_to_string(c.block())),
69            TableCell::new(unicode_script_to_string(c.script())),
70            TableCell::new(
71                c.identifier_type()
72                    .map_or("Unknown Character Type", |t| char_identifier_to_string(t)),
73            ),
74            TableCell::new(get_name(c as u32)),
75        ])
76    }
77
78    fn print_char_table(rows: Vec<Row>) {
79        let mut table = Table::init(rows);
80        table.set_titles(row![
81            "Index",
82            "Char",
83            "Category",
84            "Block",
85            "Script",
86            "Identifier Type",
87            "Name"
88        ]);
89        table.set_format(*FORMAT_CLEAN);
90        table.printstd();
91    }
92
93    /// Detect dangerous characters (only available on RawInput)
94    pub fn detect_dangerous_chars(self) {
95        // TODO allow user selection of restriction level
96        if !self.text.check_restriction_level(ASCIIOnly) {
97            // the ASCIIOnly check is too restrictive as it is intended for identifiers
98            let char_names: Vec<_> = skeleton(&self.text)
99                .enumerate()
100                .flat_map(|(i, c)| {
101                    if !(c.is_ascii_graphic() || c.is_ascii_whitespace()) {
102                        Some(Self::character_info(&i, c))
103                    } else {
104                        None
105                    }
106                })
107                .collect();
108            // we'll only throw an error if we found non-graphic, non-whitespace characters beyond the ASCII range
109            if !char_names.is_empty() {
110                eprintln!("String has restricted characters!");
111                Self::print_char_table(char_names);
112                exit(2)
113            }
114        }
115        // otherwise the string should be safe
116        // TODO fix bug detecting strings like "Æneid"
117        eprintln!("String is safe");
118        println!("{}", self.text);
119        exit(0)
120    }
121
122    /// Show character info (only available on RawInput)
123    pub fn show_character_info(self) -> String {
124        let rows: Vec<Row> = skeleton(&self.text)
125            .enumerate()
126            .map(|(i, c)| Self::character_info(&i, c))
127            .collect();
128        Self::print_char_table(rows);
129        exit(2)
130    }
131}
132
133impl UnicodeString<string_states::CleanedText> {
134    /// Strip HTML tags
135    pub fn strip_html(self) -> Self {
136        let text = Builder::new()
137            .tags(HashSet::default())
138            .clean(&self.text)
139            .to_string();
140        Self {
141            text,
142            _marker: PhantomData,
143        }
144    }
145
146    /// De-fang hyperlinks
147    pub fn defang_links(self) -> Self {
148        // let mut text = String::new();
149        let text = LinkFinder::new()
150            .spans(&self.text)
151            .map(|span| match span.kind() {
152                Some(link_kind) => match link_kind {
153                    linkify::LinkKind::Url => {
154                        // TODO use better "defanging logic"
155                        span.as_str()
156                            .replace("ftp", "fXp")
157                            .replace("http", "hXXp")
158                            .replace('.', "[.]")
159                    }
160                    linkify::LinkKind::Email => {
161                        span.as_str().replace('@', "[@]").replace('.', "[.]")
162                    }
163                    // linkify has the LinkKind enum marked as non-exhaustive so we have to have this catch-all
164                    _ => unimplemented!("unsupported link type encountered"),
165                },
166                None => span.as_str().to_string(),
167            })
168            .collect();
169        Self {
170            text,
171            _marker: PhantomData,
172        }
173    }
174
175    /// Censor profanity
176    pub fn censor_profanity(self) -> Self {
177        Self {
178            text: self.text.censor(),
179            _marker: PhantomData,
180        }
181    }
182
183    /// Convert to sluggified text
184    pub fn sluggify(self) -> Self {
185        Self {
186            text: Slugifier::default().slugify(self.text),
187            _marker: PhantomData,
188        }
189    }
190
191    /// Get the inner text
192    pub fn into_string(self) -> String {
193        self.text
194    }
195}
eunicode/unicode_string.rs

eunicode/
unicode_string.rs
