unobtanium-segmenter 0.5.2

A text segmentation toolbox for search applications inspired by charabia and tantivy.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

// SPDX-FileCopyrightText: 2026 Slatian
//
// SPDX-License-Identifier: LGPL-3.0-only

//! Things that split tokens into one or more subtokens.

mod decomposition_aho_corasick;
mod decomposition_fst;
mod lingua;
mod lingua_with_sentence_splitter;
mod naive_word;
mod segmenter;
mod unicode_sentence;
mod unicode_word;

pub use decomposition_aho_corasick::DecompositionAhoCorasick;
pub use decomposition_fst::DecompositionFst;
pub use lingua::LinguaLanguageBlockSplitter;
pub use lingua_with_sentence_splitter::LinguaLanguageBlockSentenceSplitter;
pub use naive_word::NaiveWordSplitter;
pub use segmenter::Segmenter;
pub use unicode_sentence::UnicodeSentenceSplitter;
pub use unicode_word::UnicodeWordSplitter;