sentencepiece-rust 0.1.0

From-scratch pure-Rust SentencePiece inference (encode/decode), differentially verified against the upstream C++/Python implementation.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

//! Error type for the crate.

use std::fmt;

/// Errors produced while loading a model or encoding/decoding text.
#[derive(Debug)]
pub enum Error {
    /// The model file/bytes could not be parsed as a `ModelProto`.
    Proto(String),
    /// The model parsed but is missing data required for inference.
    Model(String),
    /// A feature is recognised but not yet implemented in this port.
    Unsupported(&'static str),
    /// Underlying I/O error while reading a model file.
    Io(std::io::Error),
}

impl fmt::Display for Error {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            Error::Proto(m) => write!(f, "proto parse error: {m}"),
            Error::Model(m) => write!(f, "invalid model: {m}"),
            Error::Unsupported(m) => write!(f, "unsupported: {m}"),
            Error::Io(e) => write!(f, "io error: {e}"),
        }
    }
}

impl std::error::Error for Error {
    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
        match self {
            Error::Io(e) => Some(e),
            _ => None,
        }
    }
}

impl From<std::io::Error> for Error {
    fn from(e: std::io::Error) -> Self {
        Error::Io(e)
    }
}

pub type Result<T> = std::result::Result<T, Error>;