scirs2-text 0.4.2

Text processing module for SciRS2 (scirs2-text)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16

//! Advanced tokenization algorithms.
//!
//! - [`bpe`]: Byte-Pair Encoding tokenizer with training support.
//! - [`wordpiece`]: BERT-style WordPiece tokenizer.

pub mod bpe;
pub mod byte_level_bpe;
pub mod hf_json;
pub mod llama;
pub mod multilingual_bpe;
pub mod unicode_bpe;
pub mod unicode_normalizer;
pub mod wordpiece;

pub use bpe::{compute_merges, BpeTokenizer, BpeVocab};
pub use wordpiece::{BasicTokenizer, WordPieceTokenizer};