slm_inference 0.1.1

Backend-agnostic trait layer for running Small Language Model (SLM) inference in Rust.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117

mod gemma4;
mod llama3;
mod mistal;
mod phi4;
mod qwen25;

pub use gemma4::{Gemma4Flavor, GemmaFormatter};
pub use llama3::Llama3Formatter;
pub use mistal::{MistralFlavor, MistralFormatter};
pub use phi4::Phi4Formatter;
pub use qwen25::Qwen25Formatter;

use crate::errors::ModelFormatterError;
use crate::formatter::SlmToolStyle;
use crate::{SlmFormatter, SlmRole};

pub enum SlmDynamicFormatter {
    Gemma(GemmaFormatter),
    Llama3(Llama3Formatter),
    Mistral(MistralFormatter),
    Qwen25(Qwen25Formatter),
    Phi4(Phi4Formatter),
}

impl TryFrom<&str> for SlmDynamicFormatter {
    type Error = ModelFormatterError;

    fn try_from(value: &str) -> Result<Self, Self::Error> {
        match value.to_lowercase().as_str() {
            "llama3" => Ok(Self::Llama3(Llama3Formatter)),
            "gemma4" => Ok(Self::Gemma(GemmaFormatter::new(
                Gemma4Flavor::Vanilla,
                true,
            ))),
            "gemma4-google" => Ok(Self::Gemma(GemmaFormatter::new(
                Gemma4Flavor::GoogleOfficial,
                true,
            ))),
            "gemma4-unsloth" => Ok(Self::Gemma(GemmaFormatter::new(
                Gemma4Flavor::UnslothFixed,
                true,
            ))),
            "mistral" => Ok(Self::Mistral(MistralFormatter::new(
                MistralFlavor::V3Tekken,
                true,
            ))),
            "mistral-legacy" => Ok(Self::Mistral(MistralFormatter::new(
                MistralFlavor::Legacy,
                true,
            ))),
            "qwen25" => Ok(Self::Qwen25(Qwen25Formatter::new(true))),
            "phi4" => Ok(Self::Phi4(Phi4Formatter::new(true))),
            _ => Err(ModelFormatterError::UnknownModelFormatter(
                value.to_string(),
            )),
        }
    }
}

impl SlmDynamicFormatter {
    fn deref(&self) -> &dyn SlmFormatter {
        match self {
            Self::Llama3(f) => f,
            Self::Gemma(f) => f,
            Self::Mistral(f) => f,
            Self::Qwen25(f) => f,
            Self::Phi4(f) => f,
        }
    }
}

impl SlmFormatter for SlmDynamicFormatter {
    fn bos(&self) -> Option<&str> {
        self.deref().bos()
    }

    fn turn_start(&self, role: &SlmRole) -> String {
        self.deref().turn_start(role)
    }

    fn turn_end(&self, role: &SlmRole) -> String {
        self.deref().turn_end(role)
    }

    fn reasoning_bounds(&self) -> Option<(&str, &str)> {
        self.deref().reasoning_bounds()
    }

    fn wrap_reasoning(&self, content: &str) -> String {
        self.deref().wrap_reasoning(content)
    }

    fn reasoning_trigger(&self) -> Option<&str> {
        self.deref().reasoning_trigger()
    }

    fn tool_style(&self) -> SlmToolStyle {
        self.deref().tool_style()
    }

    fn format_tool_call(&self, name: &str, arguments_json: &str) -> String {
        self.deref().format_tool_call(name, arguments_json)
    }

    fn format_tool_response(&self, tool_name: &str, response_content: &str) -> String {
        self.deref()
            .format_tool_response(tool_name, response_content)
    }

    fn clean(&self, text: &str) -> String {
        self.deref().clean(text)
    }

    fn strip_tags(&self, text: &str) -> String {
        self.deref().strip_tags(text)
    }
}