llama-runner 2.3.2

A straightforward Rust library for running llama.cpp models locally on device
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

use llama_cpp_2::{
    ApplyChatTemplateError,
    model::{LlamaChatMessage, LlamaChatTemplate, LlamaModel},
};

use crate::MessageRole;

pub trait ChatTemplate {
    type Error;
    fn apply_template(
        &self,
        model: &LlamaModel,
        model_tmpl: &LlamaChatTemplate,
        messages: &[(MessageRole, String)],
    ) -> Result<String, Self::Error>;
}

#[derive(Clone, Default)]
pub struct ModelChatTemplate;

impl ChatTemplate for ModelChatTemplate {
    fn apply_template(
        &self,
        model: &LlamaModel,
        model_tmpl: &LlamaChatTemplate,
        messages: &[(MessageRole, String)],
    ) -> Result<String, Self::Error> {
        let llama_msg = messages
            .iter()
            .map(|(role, cnt)| LlamaChatMessage::new(role.to_string(), cnt.clone()))
            .collect::<Result<Vec<_>, _>>()
            .expect("message preprocessing failed");
        return model.apply_chat_template(model_tmpl, &llama_msg, true);
    }

    type Error = ApplyChatTemplateError;
}