rehuman 0.1.3

Unicode-safe text cleaning & typographic normalization for Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

//! Basic example showing manual `CleaningOptions` usage.

use rehuman::{
    CleaningOptions, EmojiPolicy, LineEndingStyle, TextCleaner, UnicodeNormalizationMode,
};

fn main() {
    let options = CleaningOptions {
        normalize_quotes: true,
        normalize_dashes: true,
        normalize_other: true,
        normalize_spaces: true,
        unicode_normalization: UnicodeNormalizationMode::NFKC,
        keyboard_only: true,
        emoji_policy: EmojiPolicy::Keep,
        remove_control_chars: true,
        remove_trailing_whitespace: true,
        collapse_whitespace: true,
        normalize_line_endings: Some(LineEndingStyle::Lf),
        ..CleaningOptions::default()
    };

    let cleaner = TextCleaner::new(options);
    let input = "“Hello — world…”\u{00A0}😀\r\nLine 2   \r\n";
    let out = cleaner.clean(input);
    println!("{}", out.text);
    println!("{:?}", out.stats);
}