txt_cleaner/
lib.rs

1/// Options for `txt-cleaner` operations.
2#[derive(Debug, Clone, Copy, PartialEq, Eq)]
3pub struct CleanOptions {
4    /// When true, preserves paragraph structure by collapsing consecutive blank
5    /// lines to a single blank line (`\n\n`).
6    pub preserve_paragraphs: bool,
7
8    /// When true, remove common markdown/html artifacts from the edges of the text.
9    pub strip_markdown_artifacts: bool,
10
11    /// When true, remove invisible characters like BOM and zero-width spaces.
12    pub strip_invisible_chars: bool,
13
14    /// When true, collapse all whitespace sequences to a single space.
15    pub collapse_all_whitespace: bool,
16}
17
18impl CleanOptions {
19    /// Create a new builder for `CleanOptions`.
20    pub fn builder() -> CleanBuilder {
21        CleanBuilder::default()
22    }
23}
24
25impl Default for CleanOptions {
26    fn default() -> Self {
27        Self {
28            preserve_paragraphs: true,
29            strip_markdown_artifacts: true,
30            strip_invisible_chars: true,
31            collapse_all_whitespace: false,
32        }
33    }
34}
35
36/// Builder for cleaning options.
37#[derive(Debug, Clone, Copy, PartialEq, Eq)]
38pub struct CleanBuilder {
39    preserve_paragraphs: bool,
40    strip_markdown_artifacts: bool,
41    strip_invisible_chars: bool,
42    collapse_all_whitespace: bool,
43}
44
45impl Default for CleanBuilder {
46    fn default() -> Self {
47        Self {
48            preserve_paragraphs: true,
49            strip_markdown_artifacts: true,
50            strip_invisible_chars: true,
51            collapse_all_whitespace: false,
52        }
53    }
54}
55
56impl CleanBuilder {
57    pub fn new() -> Self {
58        Self::default()
59    }
60
61    pub fn preserve_paragraphs(mut self, preserve: bool) -> Self {
62        self.preserve_paragraphs = preserve;
63        self
64    }
65
66    pub fn strip_markdown_artifacts(mut self, strip: bool) -> Self {
67        self.strip_markdown_artifacts = strip;
68        self
69    }
70
71    pub fn strip_invisible_chars(mut self, strip: bool) -> Self {
72        self.strip_invisible_chars = strip;
73        self
74    }
75
76    pub fn collapse_all_whitespace(mut self, collapse: bool) -> Self {
77        self.collapse_all_whitespace = collapse;
78        self
79    }
80
81    pub fn build(self) -> CleanOptions {
82        CleanOptions {
83            preserve_paragraphs: self.preserve_paragraphs,
84            strip_markdown_artifacts: self.strip_markdown_artifacts,
85            strip_invisible_chars: self.strip_invisible_chars,
86            collapse_all_whitespace: self.collapse_all_whitespace,
87        }
88    }
89
90    pub fn clean(self, text: &str) -> String {
91        clean_with_options(text, self.build())
92    }
93}
94
95/// Clean text with the default safe options.
96///
97/// This includes:
98/// - removing BOM and zero-width characters,
99/// - stripping common markdown/html edge artifacts,
100/// - normalizing whitespace and collapsing consecutive spaces/tabs/newlines.
101pub fn clean(text: &str) -> String {
102    clean_with_options(text, CleanOptions::default())
103}
104
105/// Clean text using a builder configuration.
106pub fn clean_with_builder(text: &str, builder: CleanBuilder) -> String {
107    builder.clean(text)
108}
109
110/// Clean text using explicit options.
111pub fn clean_with_options(text: &str, options: CleanOptions) -> String {
112    let mut result = text.to_string();
113
114    if options.strip_invisible_chars {
115        result = trim_bom_and_zero_width(&result);
116    }
117
118    if options.strip_markdown_artifacts {
119        result = trim_html_markdown_artifacts(&result);
120    }
121
122    if options.collapse_all_whitespace {
123        collapse_whitespace_to_single_space(&result)
124    } else if options.preserve_paragraphs {
125        trim_consecutive_whitespaces(&result)
126    } else {
127        collapse_whitespace_to_single_space(&result)
128    }
129}
130
131/// Remove BOM and zero-width characters from the input text.
132pub fn trim_bom_and_zero_width(text: &str) -> String {
133    const REMOVED: [char; 5] = ['\u{feff}', '\u{200b}', '\u{200c}', '\u{200d}', '\u{2060}'];
134
135    text.chars()
136        .filter(|ch| !REMOVED.contains(ch))
137        .collect()
138}
139
140/// Collapse whitespace while preserving paragraph structure.
141///
142/// - consecutive spaces/tabs become a single space
143/// - a single newline stays a newline
144/// - multiple blank lines collapse to a single blank line (`\n\n`)
145/// - leading/trailing whitespace is removed
146pub fn trim_consecutive_whitespaces(text: &str) -> String {
147    let normalized = normalize_newlines(text);
148    let mut output = String::with_capacity(normalized.len());
149    let mut newline_count = 0;
150    let mut pending_space = false;
151
152    for ch in normalized.chars() {
153        match ch {
154            '\n' => {
155                newline_count += 1;
156                pending_space = false;
157            }
158            ' ' | '\t' => {
159                if newline_count == 0 {
160                    pending_space = true;
161                }
162            }
163            _ => {
164                if newline_count > 0 {
165                    if !output.ends_with('\n') {
166                        if newline_count == 1 {
167                            output.push('\n');
168                        } else {
169                            output.push_str("\n\n");
170                        }
171                    }
172                    newline_count = 0;
173                } else if pending_space && !output.ends_with(' ') && !output.ends_with('\n') {
174                    output.push(' ');
175                }
176
177                pending_space = false;
178                output.push(ch);
179            }
180        }
181    }
182
183    output.trim_matches(|c: char| c.is_whitespace()).to_string()
184}
185
186fn collapse_whitespace_to_single_space(text: &str) -> String {
187    let mut output = String::with_capacity(text.len());
188    let mut pending_space = false;
189
190    for ch in normalize_newlines(text).chars() {
191        if ch.is_whitespace() {
192            pending_space = true;
193            continue;
194        }
195
196        if pending_space && !output.is_empty() {
197            output.push(' ');
198        }
199
200        pending_space = false;
201        output.push(ch);
202    }
203
204    output.trim().to_string()
205}
206
207fn normalize_newlines(text: &str) -> String {
208    text.replace("\r\n", "\n").replace('\r', "\n")
209}
210
211/// Remove stray markdown/html artifacts from the text edges.
212///
213/// This is heuristic cleanup for broken input such as:
214/// - `** broken sentence`
215/// - `# heading text`
216/// - `<div>lorem ipsum</div>`
217/// - stray backticks, asterisks, underscores, or tildes at the text edges
218pub fn trim_html_markdown_artifacts(text: &str) -> String {
219    let mut current = text.trim().to_string();
220
221    loop {
222        let before = current.clone();
223        current = strip_edge_artifacts(&current);
224        current = current.trim().to_string();
225        if current == before {
226            break;
227        }
228    }
229
230    current
231}
232
233fn strip_edge_artifacts(text: &str) -> String {
234    if let Some(stripped) = strip_html_tag_edge(text) {
235        return stripped;
236    }
237
238    if let Some(stripped) = strip_heading_prefix(text) {
239        return stripped.trim_start().to_string();
240    }
241
242    if let Some(stripped) = strip_blockquote_or_list_prefix(text) {
243        return stripped.trim_start().to_string();
244    }
245
246    if let Some(stripped) = strip_unpaired_edge_markers(text, true) {
247        return stripped;
248    }
249
250    if let Some(stripped) = strip_unpaired_edge_markers(text, false) {
251        return stripped;
252    }
253
254    if let Some(stripped) = strip_matching_wrappers(text) {
255        return stripped.trim().to_string();
256    }
257
258    text.to_string()
259}
260
261fn strip_html_tag_edge(text: &str) -> Option<String> {
262    let lower = text.to_lowercase();
263
264    let html_tags = ["<div>", "<p>", "<span>", "<strong>", "<em>", "<b>", "<i>"];
265    for tag in html_tags {
266        if lower.starts_with(tag) {
267            return Some(text[tag.len()..].trim_start().to_string());
268        }
269    }
270
271    let trimmed_end = text.trim_end();
272    let lower_end = trimmed_end.to_lowercase();
273    for tag in html_tags {
274        if lower_end.ends_with(tag) {
275            return Some(trimmed_end[..trimmed_end.len() - tag.len()].trim_end().to_string());
276        }
277    }
278
279    let closing_tags = ["</div>", "</p>", "</span>", "</strong>", "</em>", "</b>", "</i>"];
280    for tag in closing_tags {
281        if lower.starts_with(tag) {
282            return Some(text[tag.len()..].trim_start().to_string());
283        }
284    }
285
286    for tag in closing_tags {
287        if lower_end.ends_with(tag) {
288            return Some(trimmed_end[..trimmed_end.len() - tag.len()].trim_end().to_string());
289        }
290    }
291
292    if lower.starts_with("<!--") {
293        return Some(text[4..].trim_start().to_string());
294    }
295
296    if lower_end.ends_with("-->") {
297        return Some(trimmed_end[..trimmed_end.len() - 3].trim_end().to_string());
298    }
299
300    None
301}
302
303fn strip_blockquote_or_list_prefix(text: &str) -> Option<String> {
304    let trimmed = text.trim_start();
305
306    if trimmed.starts_with("> ") {
307        return Some(trimmed[2..].to_string());
308    }
309
310    if trimmed == ">" {
311        return Some(String::new());
312    }
313
314    for prefix in ["- ", "* ", "+ "] {
315        if trimmed.starts_with(prefix) {
316            return Some(trimmed[prefix.len()..].to_string());
317        }
318    }
319
320    None
321}
322
323fn strip_matching_wrappers(text: &str) -> Option<&str> {
324    const MARKERS: [char; 4] = ['*', '_', '~', '`'];
325
326    for marker in MARKERS {
327        let prefix = count_leading_chars(text, marker);
328        let suffix = count_trailing_chars(text, marker);
329
330        if prefix > 0 && suffix > 0 {
331            let matched = prefix.min(suffix);
332            if matched > 0 {
333                let start = text.char_indices().nth(matched).map(|(idx, _)| idx).unwrap_or(text.len());
334                let end = text.len() - text.chars().rev().take(matched).map(|c| c.len_utf8()).sum::<usize>();
335                if start < end {
336                    return Some(&text[start..end]);
337                }
338            }
339        }
340    }
341
342    None
343}
344
345fn strip_unpaired_edge_markers(text: &str, leading: bool) -> Option<String> {
346    const MARKERS: [char; 4] = ['*', '_', '~', '`'];
347
348    if leading {
349        let count = count_leading_chars_set(text, &MARKERS);
350        if count > 0 {
351            let after = &text[text.char_indices().nth(count).map(|(idx, _)| idx).unwrap_or(text.len())..];
352            if after.is_empty()
353                || after.chars().next().map_or(false, |c| {
354                    c.is_whitespace() || c == '#' || c == '>' || c == '<'
355                })
356            {
357                return Some(after.trim_start().to_string());
358            }
359        }
360    } else {
361        let count = count_trailing_chars_set(text, &MARKERS);
362        if count > 0 {
363            let before_end = text.char_indices().rev().nth(count - 1).map(|(idx, _ch)| idx).unwrap_or(0);
364            let before = &text[..before_end];
365            if before.is_empty()
366                || before.chars().rev().next().map_or(false, |c| {
367                    c.is_whitespace() || c == '#' || c == '>' || c == '<'
368                })
369            {
370                return Some(before.trim_end().to_string());
371            }
372        }
373    }
374
375    None
376}
377
378fn count_leading_chars(text: &str, marker: char) -> usize {
379    text.chars().take_while(|&c| c == marker).count()
380}
381
382fn count_trailing_chars(text: &str, marker: char) -> usize {
383    text.chars().rev().take_while(|&c| c == marker).count()
384}
385
386fn count_leading_chars_set(text: &str, markers: &[char]) -> usize {
387    text.chars().take_while(|c| markers.contains(c)).count()
388}
389
390fn count_trailing_chars_set(text: &str, markers: &[char]) -> usize {
391    text.chars().rev().take_while(|c| markers.contains(c)).count()
392}
393
394fn strip_heading_prefix(text: &str) -> Option<&str> {
395    for level in (1..=6).rev() {
396        let prefix = "#".repeat(level);
397        if let Some(stripped) = text.strip_prefix(&prefix) {
398            if stripped.starts_with(char::is_whitespace) {
399                return Some(stripped);
400            }
401        }
402    }
403
404    None
405}
406
407#[cfg(test)]
408mod tests {
409    use super::*;
410
411    #[test]
412    fn trim_bom_and_zero_width_removes_invisible_chars() {
413        let input = "\u{feff}Hello\u{200B} world\u{200C}!";
414        assert_eq!(trim_bom_and_zero_width(input), "Hello world!");
415    }
416
417    #[test]
418    fn trim_consecutive_whitespaces_collapses_space_tab_newline() {
419        let input = "  foo   bar\t\tbaz\n\n\nqux   ";
420        assert_eq!(trim_consecutive_whitespaces(input), "foo bar baz\n\nqux");
421    }
422
423    #[test]
424    fn collapse_whitespace_to_single_space_works() {
425        let input = "foo\n\nbar\t\t baz";
426        assert_eq!(collapse_whitespace_to_single_space(input), "foo bar baz");
427    }
428
429    #[test]
430    fn trim_html_markdown_artifacts_removes_edge_tokens() {
431        let input = "  **# Hello *world* <div>  ";
432        assert_eq!(trim_html_markdown_artifacts(input), "Hello *world*");
433    }
434
435    #[test]
436    fn trim_html_markdown_artifacts_removes_heading_and_html_wrappers() {
437        let input = "## <div>**Hello**</div>";
438        assert_eq!(trim_html_markdown_artifacts(input), "Hello");
439    }
440
441    #[test]
442    fn clean_with_builder_can_collapse_all_whitespace() {
443        let input = "foo\n\nbar\t baz";
444        let output = CleanBuilder::new().collapse_all_whitespace(true).clean(input);
445        assert_eq!(output, "foo bar baz");
446    }
447
448    #[test]
449    fn builder_api_allows_full_configuration() {
450        let input = "\u{feff}  **# Hello  \n\n\n  world!  **\u{200B} ";
451        let options = CleanOptions::builder()
452            .strip_invisible_chars(true)
453            .strip_markdown_artifacts(true)
454            .preserve_paragraphs(false)
455            .collapse_all_whitespace(true)
456            .build();
457
458        assert_eq!(clean_with_options(input, options), "Hello world!");
459    }
460
461    #[test]
462    fn clean_applies_all_steps() {
463        let input = "\u{feff}**\n  Hello   \n\n\n  world!  **\u{200B} ";
464        assert_eq!(clean(input), "Hello\n\nworld!");
465    }
466}
txt_cleaner/lib.rs

txt_cleaner/
lib.rs
