slm_inference 0.1.0

Backend-agnostic trait layer for running Small Language Model (SLM) inference in Rust.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103

use std::os::raw::c_int;
use std::string::FromUtf8Error;

#[derive(Debug, Eq, PartialEq, thiserror::Error)]
pub enum DecodeError {
    #[error("Decode Error: NoKvCacheSlot")]
    NoKvCacheSlot,
    #[error("Decode Error: no tokens")]
    NTokensZero,
    #[error("Decode Error:  {0}")]
    Unknown(c_int),
}

impl From<i32> for DecodeError {
    fn from(value: i32) -> Self {
        match value {
            1 => DecodeError::NoKvCacheSlot,
            -1 => DecodeError::NTokensZero,
            i => DecodeError::Unknown(i),
        }
    }
}

#[derive(Debug, Eq, PartialEq, thiserror::Error)]
pub enum SamplingError {}

#[derive(Debug, Eq, PartialEq, thiserror::Error)]
pub enum TokenToStringError {
    #[error("Unknown Token Type")]
    UnknownTokenType,
    #[error("Insufficient Buffer Space {0}")]
    InsufficientBufferSpace(c_int),
    #[error("FromUtf8Error {0}")]
    FromUtf8Error(#[from] FromUtf8Error),
    #[error("Invalid Lstrip")]
    InvalidLstrip,
}

#[derive(Debug, Eq, PartialEq, thiserror::Error)]
pub enum StringToTokenError {
    #[error("FromUtf8Error {0}")]
    FromUtf8Error(#[from] FromUtf8Error),
    #[error("FfiError {0}")]
    FfiError(#[from] FfiError),
}

#[derive(Debug, Eq, PartialEq, thiserror::Error)]
pub enum InferenceError {
    #[error("General Error {0}")]
    GeneralError(String),
    #[error("Batch Error {0}")]
    BatchError(#[from] BatchError),
    #[error("Ffi Error {0}")]
    FfiError(#[from] FfiError),
    #[error("StringToTokenError {0}")]
    StringToTokenError(#[from] StringToTokenError),
    #[error("TokenToStringError {0}")]
    TokenToStringError(#[from] TokenToStringError),
    #[error("DecodeError {0}")]
    DecodeError(#[from] DecodeError),
    #[error("SamplingError {0}")]
    SamplingError(#[from] SamplingError),
}

#[derive(Debug, Eq, PartialEq, thiserror::Error)]
pub enum GgufLoaderError {
    #[error("Invalid Path")]
    InvalidPath,
    #[error("Bad model")]
    BadModel,
    #[error("Ffi Error {0}")]
    FfiError(#[from] FfiError),
}

#[derive(thiserror::Error, Debug, PartialEq, Eq)]
pub enum BatchError {
    #[error("Insufficient Space of {0}")]
    InsufficientSpace(usize),
    #[error("Empty buffer")]
    EmptyBuffer,
    #[error("n_tokens {0} is too large for a batch")]
    NtokTooLarge(usize),
    #[error("n_seq_max {0} is too large for a batch")]
    NseqTooLarge(usize),
}

#[derive(thiserror::Error, Debug, PartialEq, Eq)]
pub enum FfiError {
    #[error("Null Ptr")]
    NullPtr,
    #[error("Cstring Allocation Error")]
    CstAllocationError,
    #[error("C_int Conversion Error")]
    CintConversionError,
    #[error("General Error {0}")]
    GeneralError(String),
}

#[derive(thiserror::Error, Debug, PartialEq, Eq)]
pub enum ContextBuilderError {
    #[error("Ffi Error {0}")]
    FfiError(#[from] FfiError),
}