llama-runner 2.3.2

A straightforward Rust library for running llama.cpp models locally on device
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

mod ext;
mod gemma3;
mod gemma4;
mod msg;
mod req;
mod stream;

use std::sync::LazyLock;

pub use ext::*;
pub use gemma3::*;
pub use gemma4::*;
use llama_cpp_2::llama_backend::LlamaBackend;
pub use msg::*;
pub use req::*;
pub use stream::*;

use crate::{error::GenericRunnerError, template::ChatTemplate};

static LLAMA_BACKEND: LazyLock<LlamaBackend> = LazyLock::new(|| {
    llama_cpp_2::send_logs_to_tracing(llama_cpp_2::LogOptions::default());
    LlamaBackend::init().unwrap()
});

pub trait TextLmRunner<'s, 'req, Tmpl>
where
    Tmpl: ChatTemplate,
{
    fn stream_lm_response(
        &'s self,
        request: GenericTextLmRequest<'req, Tmpl>,
    ) -> impl Iterator<Item = Result<String, GenericRunnerError<Tmpl::Error>>>;
}

pub trait VisionLmRunner<'s, 'req, Tmpl>
where
    Tmpl: ChatTemplate,
{
    fn stream_vlm_response(
        &'s self,
        request: GenericVisionLmRequest<'req, Tmpl>,
    ) -> impl Iterator<Item = Result<String, GenericRunnerError<Tmpl::Error>>>;
}