llama-runner 2.3.2

A straightforward Rust library for running llama.cpp models locally on device
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54

use llama_cpp_2::{GrammarError, model::LlamaModel, sampling::LlamaSampler};
use serde::{Deserialize, Serialize};
use strum::Display;

#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct SimpleSamplingParams {
    pub top_p: Option<f32>,
    pub top_k: Option<i32>,
    pub temperature: Option<f32>,
    pub seed: Option<u32>,
    pub presence_penalty: Option<f32>,
    pub repetition_penalty: Option<f32>,
}

impl SimpleSamplingParams {
    pub fn to_llama(&self) -> LlamaSampler {
        let mut samplers = Vec::new();
        if let Some(k) = self.top_k {
            samplers.push(LlamaSampler::top_k(k));
        }
        if let Some(p) = self.top_p {
            samplers.push(LlamaSampler::top_p(p, 8));
        }
        if let Some(p) = self.presence_penalty
            && let Some(r) = self.repetition_penalty
        {
            samplers.push(LlamaSampler::penalties(-1, p, 0.0, r));
        }
        samplers.push(LlamaSampler::dist(
            self.seed.unwrap_or_else(|| rand::random()),
        ));
        LlamaSampler::chain_simple(samplers)
    }
}

#[derive(Debug, Clone)]
pub struct LlguidanceSamplingParams {
    pub schema: LlguidanceSchema,
    pub data: String,
}

impl LlguidanceSamplingParams {
    pub fn to_llama(&self, model: &LlamaModel) -> Result<LlamaSampler, GrammarError> {
        LlamaSampler::llguidance(model, self.schema.to_string().as_str(), &self.data)
    }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Display)]
pub enum LlguidanceSchema {
    #[strum(to_string = "json")]
    Json,
    #[strum(to_string = "lark")]
    Lark,
}