callm 0.2.0

Run Generative AI models directly on your hardware
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

//! This module provides loaders for different model formats.

pub mod gguf;
pub use gguf::LoaderGguf;
pub mod safetensors;
pub use safetensors::LoaderSafetensors;

use crate::device::DeviceConfig;
use crate::error::CallmError;
use crate::models::ModelImpl;
use crate::templates::TemplateImpl;
use std::sync::{Arc, Mutex};
use tokenizers::tokenizer::Tokenizer;

/// A trait for defining the interface of model loaders.
pub trait LoaderImpl: Send {
    /// Sets the device configuration for the loader.
    fn set_device(&mut self, device: Arc<DeviceConfig>);

    /// Loads the model and returns it wrapped in an `Arc<Mutex<dyn ModelImpl>>`.
    fn load(&mut self) -> Result<Arc<Mutex<dyn ModelImpl>>, CallmError>;

    /// Returns the tokenizer associated with the model.
    fn tokenizer(&mut self) -> Result<Tokenizer, CallmError>;

    /// Returns the template associated with the model.
    fn template(&mut self) -> Result<Box<dyn TemplateImpl>, CallmError>;
}