normy 0.1.4

Ultra-fast, zero-copy text normalization for Rust NLP pipelines & tokenizers
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

#![forbid(unsafe_code)]
#![cfg_attr(docsrs, feature(doc_cfg))]

pub mod context;
pub mod lang;
pub mod process;
pub mod stage;
pub mod testing;

pub use lang::data::*;
pub use normy::{DynamicNormyBuilder, Normy, NormyBuilder, NormyError};

pub use stage::case_fold::CaseFold;
pub use stage::lower_case::LowerCase;
pub use stage::normalization::{NFC, NFD, NFKC, NFKD};
pub use stage::normalize_punctuation::NormalizePunctuation;
pub use stage::normalize_whitespace::{
    COLLAPSE_WHITESPACE, COLLAPSE_WHITESPACE_UNICODE, NORMALIZE_WHITESPACE_FULL, TRIM_WHITESPACE,
    TRIM_WHITESPACE_UNICODE,
};
pub use stage::remove_diacritics::RemoveDiacritics;
pub use stage::segment_words::SegmentWords;
pub use stage::strip_control_chars::StripControlChars;
pub use stage::strip_format_controls::StripFormatControls;
pub use stage::strip_html::StripHtml;
pub use stage::strip_markdown::StripMarkdown;
pub use stage::transliterate::Transliterate;
pub use stage::unify_width::UnifyWidth;

// Internal only
mod normy;
mod unicode;

#[cfg(test)]
mod tests {
    include!("tests/integration.rs");
    include!("tests/proptest.rs");
}