sipp-rs 0.1.0

Unified Rust library for extensible Sipp inference
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

//! Native-runtime convenience methods on `InferenceRuntime`.

use crate::error::Result;
use crate::native_bridge;

use super::InferenceRuntime;

/////////////////////////////////////////////////////////////////////////////////
/// TESTS
/////////////////////////////////////////////////////////////////////////////////

#[cfg(test)]
#[path = "../../tests/runtime/inference_runtime/native_tests.rs"]
mod native_tests;

/////////////////////////////////////////////////////////////////////////////////
/// SRC
/////////////////////////////////////////////////////////////////////////////////

impl InferenceRuntime {
    pub fn get_bos_text(&self) -> Result<String> {
        let bos = self.native_runtime.bos_token();
        if bos == native_bridge::LLAMA_TOKEN_NULL {
            return Ok(String::new());
        }
        self.native_runtime.token_to_piece(bos, true)
    }

    pub fn get_eos_text(&self) -> Result<String> {
        let eos = self.native_runtime.eos_token();
        if eos == native_bridge::LLAMA_TOKEN_NULL {
            return Ok(String::new());
        }
        self.native_runtime.token_to_piece(eos, true)
    }

    pub fn chat_template_source(&self) -> Result<Option<String>> {
        self.native_runtime.chat_template_source().map(Some)
    }

    pub fn apply_chat_template_json(
        &self,
        messages_json: &str,
        add_assistant: bool,
    ) -> Result<String> {
        self.native_runtime
            .apply_chat_template_json(messages_json, add_assistant)
    }

    pub fn media_marker(&self) -> Result<String> {
        Ok(native_bridge::mtmd_default_marker())
    }
}