shimmy 1.3.3

Lightweight 5MB Ollama alternative with native SafeTensors support. No Python dependencies, 2x faster loading.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137

use anyhow::Result;
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use std::path::PathBuf;

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GenOptions {
    pub max_tokens: usize,
    pub temperature: f32,
    pub top_p: f32,
    pub top_k: i32,
    pub repeat_penalty: f32,
    pub seed: Option<u32>,
    pub stream: bool,
}

impl Default for GenOptions {
    fn default() -> Self {
        Self {
            max_tokens: 256,
            temperature: 0.7,
            top_p: 0.9,
            top_k: 40,
            repeat_penalty: 1.1,
            seed: None,
            stream: true,
        }
    }
}

// Universal backend support - true shim architecture
#[derive(Debug, Clone)]
#[cfg(feature = "huggingface")]
pub enum ModelBackend {
    // GGUF via llama.cpp (existing)
    LlamaGGUF {
        base_path: PathBuf,
        lora_path: Option<PathBuf>,
    },

    // HuggingFace + PEFT (your personal models!)
    HuggingFace {
        base_model_id: String,      // "microsoft/Phi-3-mini-4k-instruct"
        peft_path: Option<PathBuf>, // "./phi3-personal-h100-cloud"
        use_local: bool,            // Use cached vs download
    },

    // Pure Rust Candle (future)
    Candle {
        model_path: PathBuf,
        adapter_path: Option<PathBuf>,
    },
}

#[derive(Debug, Clone)]
#[cfg(feature = "huggingface")]
pub struct UniversalModelSpec {
    pub name: String,
    pub backend: ModelBackend,
    pub template: Option<String>,
    pub ctx_len: usize,
    pub device: String, // "cpu", "cuda", "metal"
    pub n_threads: Option<i32>,
}

// Legacy ModelSpec for backward compatibility
#[derive(Debug, Clone)]
pub struct ModelSpec {
    pub name: String,
    pub base_path: PathBuf,
    pub lora_path: Option<PathBuf>,
    pub template: Option<String>,
    pub ctx_len: usize,
    pub n_threads: Option<i32>,
}

#[cfg(feature = "huggingface")]
impl From<ModelSpec> for UniversalModelSpec {
    fn from(spec: ModelSpec) -> Self {
        UniversalModelSpec {
            name: spec.name,
            backend: ModelBackend::LlamaGGUF {
                base_path: spec.base_path,
                lora_path: spec.lora_path,
            },
            template: spec.template,
            ctx_len: spec.ctx_len,
            device: "cpu".to_string(),
            n_threads: spec.n_threads,
        }
    }
}

// Universal Engine trait - supports any backend
#[async_trait]
#[cfg(feature = "huggingface")]
pub trait UniversalEngine: Send + Sync {
    async fn load(&self, spec: &UniversalModelSpec) -> Result<Box<dyn UniversalModel>>;
}

#[async_trait]
#[cfg(feature = "huggingface")]
pub trait UniversalModel: Send + Sync {
    async fn generate(
        &self,
        prompt: &str,
        opts: GenOptions,
        on_token: Option<Box<dyn FnMut(String) + Send>>,
    ) -> Result<String>;
}

// Legacy trait for backward compatibility
#[async_trait]
pub trait InferenceEngine: Send + Sync {
    async fn load(&self, spec: &ModelSpec) -> Result<Box<dyn LoadedModel>>;
}

#[async_trait]
pub trait LoadedModel: Send + Sync {
    async fn generate(
        &self,
        prompt: &str,
        opts: GenOptions,
        on_token: Option<Box<dyn FnMut(String) + Send>>,
    ) -> Result<String>;
}

pub mod llama;

#[cfg(feature = "huggingface")]
pub mod huggingface;

#[cfg(feature = "huggingface")]
pub mod universal;

pub mod adapter;
pub mod safetensors_native;