Skip to main content

sentencepiece_rs/
lib.rs

1//! SentencePiece runtime in Rust.
2//!
3//! This crate loads existing SentencePiece `.model` / `.spm` files and exposes
4//! a small processor API for normalization, encoding, and decoding.
5
6mod darts;
7mod error;
8mod model;
9mod normalizer;
10mod processor;
11mod proto;
12mod util;
13
14pub use crate::error::{Error, Result};
15pub use crate::model::{ModelType, Piece, PieceType, SentencePieceModel};
16pub use crate::normalizer::Normalizer;
17pub use crate::processor::SentencePieceProcessor;
18pub use crate::util::{DEFAULT_UNKNOWN_SURFACE, REPLACEMENT_CHARACTER, SPACE_SYMBOL};