sentencepiece-rs 0.2.1

Rust runtime reimplementation of SentencePiece model loading, normalization, encoding, and decoding.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18

//! SentencePiece runtime in Rust.
//!
//! This crate loads existing SentencePiece `.model` / `.spm` files and exposes
//! a small processor API for normalization, encoding, and decoding.

mod darts;
mod error;
mod model;
mod normalizer;
mod processor;
mod proto;
mod util;

pub use crate::error::{Error, Result};
pub use crate::model::{ModelType, Piece, PieceType, SentencePieceModel};
pub use crate::normalizer::Normalizer;
pub use crate::processor::SentencePieceProcessor;
pub use crate::util::{DEFAULT_UNKNOWN_SURFACE, REPLACEMENT_CHARACTER, SPACE_SYMBOL};