tllama 0.1.1

Lightweight Local LLM Inference Engine
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

use crate::discover::Model;
use anyhow::Result;

pub trait EngineBackend: Send + Sync {
    fn new(args: &EngineConfig, model: &Model) -> Result<Self>
    where
        Self: Sized;
    fn infer(
        &self,
        prompt: &str,
        option: Option<&EngineConfig>,
        callback: Option<Box<dyn FnMut(String) + Send>>,
    ) -> Result<String>;
    fn get_model_info(&self) -> Model;
}

#[macro_export]
macro_rules! def_callback {
    (|$arg:ident| $body:block) => {
        Some(Box::new(move |$arg: String| $body) as Box<dyn FnMut(String) + Send + 'static>)
    };
}

#[derive(Clone, Debug, Serialize)]
pub struct EngineConfig {
    pub n_ctx: i32,
    pub n_len: Option<u32>,
    pub temperature: f32,
    pub top_k: i32,
    pub top_p: f32,
    pub repeat_penalty: f32,
}

pub mod adapter;

pub use adapter::InferenceEngine;
use serde::Serialize;

#[cfg(feature = "engine-llama-cpp")]
pub mod llama_cpp;

#[cfg(feature = "engine-hf")]
pub mod hf;

#[cfg(not(any(feature = "engine-llama-cpp", feature = "engine-hf")))]
compile_error!(
    "No template engine feature enabled. Please enable either 'engine-llama-cpp' or 'engine-hf'."
);