language-text-analysis 0.1.3

Small, deterministic multilingual text-analysis pipeline for search and document processing
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14

//! Tokenization on Unicode word boundaries (UAX #29).

use unicode_segmentation::UnicodeSegmentation;

/// Split already-[normalized](crate) text into word tokens on Unicode
/// word boundaries.
///
/// Punctuation and whitespace segments are dropped. UAX #29 breaks
/// between CJK ideographs, so each Han character becomes its own token
/// (character-level CJK tokenization); mixed-script text segments by
/// each script's own rules.
pub(crate) fn tokenize(text: &str) -> impl Iterator<Item = &str> {
    text.unicode_words()
}