Skip to main content

rigspark_runtime/
cache.rs

1use rigspark_core::sizing::KvCacheType;
2use serde::{Deserialize, Serialize};
3use std::collections::BTreeMap;
4
5#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
6#[serde(rename_all = "lowercase")]
7pub enum FlashAttention {
8    #[default]
9    Auto,
10    On,
11    Off,
12}
13
14#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
15#[serde(rename_all = "lowercase")]
16pub enum PromptReuse {
17    #[default]
18    Off,
19    Reuse,
20}
21
22impl FlashAttention {
23    pub fn name(self) -> &'static str {
24        match self {
25            Self::Auto => "auto",
26            Self::On => "on",
27            Self::Off => "off",
28        }
29    }
30    pub fn parse(value: &str) -> Option<Self> {
31        [Self::Auto, Self::On, Self::Off]
32            .into_iter()
33            .find(|kind| kind.name() == value)
34    }
35}
36
37impl PromptReuse {
38    pub fn name(self) -> &'static str {
39        match self {
40            Self::Off => "off",
41            Self::Reuse => "reuse",
42        }
43    }
44    pub fn parse(value: &str) -> Option<Self> {
45        [Self::Off, Self::Reuse]
46            .into_iter()
47            .find(|kind| kind.name() == value)
48    }
49}
50
51#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
52#[serde(default, rename_all = "camelCase", deny_unknown_fields)]
53pub struct CacheProfile {
54    pub kv_k: KvCacheType,
55    pub kv_v: KvCacheType,
56    pub flash_attention: FlashAttention,
57    pub prompt_reuse: PromptReuse,
58}
59
60impl CacheProfile {
61    /// Rules that hold on every backend.
62    pub fn validate(&self) -> Result<(), CacheError> {
63        if self.kv_v != KvCacheType::F16 && self.flash_attention == FlashAttention::Off {
64            return Err(CacheError::Invalid(
65                "a quantized V cache requires flash attention",
66            ));
67        }
68        Ok(())
69    }
70    pub fn summary(&self) -> String {
71        let kv = if self.kv_k == self.kv_v {
72            self.kv_k.name().to_owned()
73        } else {
74            format!("K {} / V {}", self.kv_k.name(), self.kv_v.name())
75        };
76        format!(
77            "KV {kv}, flash attention {}, prompt reuse {}",
78            self.flash_attention.name(),
79            self.prompt_reuse.name()
80        )
81    }
82}
83
84/// Cache options a user asked for; unset fields inherit from the base profile.
85#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
86pub struct CacheFlags {
87    pub kv: Option<KvCacheType>,
88    pub flash_attention: Option<FlashAttention>,
89    pub prompt_reuse: Option<PromptReuse>,
90}
91
92impl CacheFlags {
93    pub fn is_empty(&self) -> bool {
94        *self == Self::default()
95    }
96    /// `base` with every given flag applied on top.
97    pub fn over(self, base: Option<CacheProfile>) -> Option<CacheProfile> {
98        if self.is_empty() {
99            return base;
100        }
101        let mut profile = base.unwrap_or_default();
102        if let Some(kind) = self.kv {
103            profile.kv_k = kind;
104            profile.kv_v = kind;
105        }
106        if let Some(flash) = self.flash_attention {
107            profile.flash_attention = flash;
108        }
109        if let Some(reuse) = self.prompt_reuse {
110            profile.prompt_reuse = reuse;
111        }
112        Some(profile)
113    }
114}
115
116/// Arguments and environment a cache provider adds to a backend launch.
117#[derive(Debug, Clone, Default, PartialEq, Eq)]
118pub struct SpawnDelta {
119    pub args: Vec<String>,
120    pub env: BTreeMap<String, String>,
121}
122
123#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
124pub enum CacheError {
125    #[error("invalid cache profile: {0}")]
126    Invalid(&'static str),
127    #[error("{backend} cannot apply this cache profile: {reason}")]
128    Unsupported {
129        backend: &'static str,
130        reason: &'static str,
131    },
132    #[error("{backend} cache settings are not verified yet; use the default cache profile")]
133    Unverified { backend: &'static str },
134}
135
136pub trait CacheProvider: Send + Sync {
137    fn name(&self) -> &'static str;
138    fn apply(
139        &self,
140        profile: &CacheProfile,
141        backend: &str,
142        owned: bool,
143    ) -> Result<SpawnDelta, CacheError>;
144}
145
146/// Providers are compiled in; names never load code or pass raw arguments through.
147pub fn provider(name: &str) -> Option<&'static dyn CacheProvider> {
148    match name {
149        "native" => Some(&NativeCacheProvider),
150        _ => None,
151    }
152}
153
154/// The profile worth recording and applying: `None` when absent or equal to the default.
155pub fn effective(profile: Option<&CacheProfile>) -> Option<CacheProfile> {
156    profile
157        .copied()
158        .filter(|profile| *profile != CacheProfile::default())
159}
160
161/// The profile a launch applies: flags win, then the running profile for `switch`
162/// or the user default for `up`, then the backend default (`None`).
163pub fn requested(
164    flags: CacheFlags,
165    switch: bool,
166    running: Option<CacheProfile>,
167    user_default: Option<CacheProfile>,
168) -> Option<CacheProfile> {
169    effective(
170        flags
171            .over(if switch { running } else { user_default })
172            .as_ref(),
173    )
174}
175
176/// Launch settings for `backend`, refusing profiles it cannot apply before anything starts.
177pub fn launch_delta(
178    profile: Option<&CacheProfile>,
179    backend: &str,
180    owned: bool,
181) -> Result<SpawnDelta, CacheError> {
182    match effective(profile) {
183        None => Ok(SpawnDelta::default()),
184        Some(profile) => NativeCacheProvider.apply(&profile, backend, owned),
185    }
186}
187
188/// Cache settings each backend already exposes (verified: llama.cpp b10090, Ollama 0.32.5).
189pub struct NativeCacheProvider;
190
191impl CacheProvider for NativeCacheProvider {
192    fn name(&self) -> &'static str {
193        "native"
194    }
195    fn apply(
196        &self,
197        profile: &CacheProfile,
198        backend: &str,
199        owned: bool,
200    ) -> Result<SpawnDelta, CacheError> {
201        if !rigspark_core::catalog::BACKENDS.contains(&backend) {
202            return Err(CacheError::Invalid("unknown backend"));
203        }
204        profile.validate()?;
205        if *profile == CacheProfile::default() {
206            return Ok(SpawnDelta::default());
207        }
208        match backend {
209            "llamacpp" => Ok(llama_cpp(profile)),
210            "ollama" => ollama(profile, owned),
211            "mlx" => Err(CacheError::Unverified { backend: "mlx" }),
212            _ => Err(CacheError::Unsupported {
213                backend: "lmstudio",
214                reason: "LM Studio is attach-only; change its cache settings in LM Studio",
215            }),
216        }
217    }
218}
219
220fn flash_enabled(profile: &CacheProfile) -> Option<bool> {
221    match profile.flash_attention {
222        FlashAttention::On => Some(true),
223        FlashAttention::Off => Some(false),
224        FlashAttention::Auto => (profile.kv_v != KvCacheType::F16).then_some(true),
225    }
226}
227
228fn llama_cpp(profile: &CacheProfile) -> SpawnDelta {
229    let mut args = Vec::new();
230    for (flag, kind) in [
231        ("--cache-type-k", profile.kv_k),
232        ("--cache-type-v", profile.kv_v),
233    ] {
234        if kind != KvCacheType::F16 {
235            args.extend([flag.to_string(), kind.name().to_string()]);
236        }
237    }
238    if let Some(enabled) = flash_enabled(profile) {
239        args.extend([
240            "--flash-attn".into(),
241            if enabled { "on" } else { "off" }.into(),
242        ]);
243    }
244    if profile.prompt_reuse == PromptReuse::Reuse {
245        args.extend(["--cache-reuse".into(), "256".into()]);
246    }
247    SpawnDelta {
248        args,
249        env: BTreeMap::new(),
250    }
251}
252
253fn ollama(profile: &CacheProfile, owned: bool) -> Result<SpawnDelta, CacheError> {
254    let unsupported = |reason| CacheError::Unsupported {
255        backend: "ollama",
256        reason,
257    };
258    if profile.kv_k != profile.kv_v {
259        return Err(unsupported("Ollama uses one cache type for both K and V"));
260    }
261    let mut env = BTreeMap::new();
262    if let Some(enabled) = flash_enabled(profile) {
263        env.insert(
264            "OLLAMA_FLASH_ATTENTION".into(),
265            if enabled { "1" } else { "0" }.into(),
266        );
267    }
268    if profile.kv_k != KvCacheType::F16 {
269        env.insert("OLLAMA_KV_CACHE_TYPE".into(), profile.kv_k.name().into());
270    }
271    // Prompt-prefix reuse is Ollama's default behaviour and needs no setting.
272    if !env.is_empty() && !owned {
273        return Err(unsupported(
274            "cache settings are daemon-wide and apply only to an Ollama daemon rigspark starts",
275        ));
276    }
277    Ok(SpawnDelta {
278        args: Vec::new(),
279        env,
280    })
281}