alith-models 0.4.3

Load and Download LLM Models, Metadata, and Tokenizers
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

use crate::{
    LLMModelBase,
    local_model::{
        LocalLLMModel,
        gguf::{load_chat_template, load_tokenizer},
        metadata::LocalLLMMetadata,
    },
};

#[derive(Default, Clone)]
pub struct GgufLocalLoader {
    pub local_quant_file_path: Option<std::path::PathBuf>,
    pub local_config_path: Option<std::path::PathBuf>,
    pub local_tokenizer_path: Option<std::path::PathBuf>,
    pub local_tokenizer_config_path: Option<std::path::PathBuf>,
    pub model_id: Option<String>,
}

impl GgufLocalLoader {
    pub fn load(&mut self) -> crate::Result<LocalLLMModel> {
        let local_model_path =
            if let Some(local_quant_file_path) = self.local_quant_file_path.as_ref() {
                local_quant_file_path.to_owned()
            } else {
                crate::bail!("local_quant_file_path must be set")
            };

        let model_id = if let Some(model_id) = &self.model_id {
            model_id.to_owned()
        } else {
            local_model_path.to_string_lossy().to_string()
        };

        let model_metadata = LocalLLMMetadata::from_gguf_path(&local_model_path)?;

        Ok(LocalLLMModel {
            model_base: LLMModelBase {
                model_id,
                model_ctx_size: model_metadata.context_length(),
                inference_ctx_size: model_metadata.context_length(),
                tokenizer: load_tokenizer(&self.local_tokenizer_path, &model_metadata)?,
            },
            chat_template: load_chat_template(&self.local_tokenizer_config_path, &model_metadata)?,
            model_metadata,
            local_model_path,
        })
    }
}