Skip to main content

rigspark_runtime/
cache.rs

1use rigspark_core::sizing::KvCacheType;
2use serde::{Deserialize, Serialize};
3use std::collections::BTreeMap;
4
5#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
6#[serde(rename_all = "lowercase")]
7pub enum FlashAttention {
8    #[default]
9    Auto,
10    On,
11    Off,
12}
13
14#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
15#[serde(rename_all = "lowercase")]
16pub enum PromptReuse {
17    #[default]
18    Off,
19    Reuse,
20}
21
22impl FlashAttention {
23    pub fn name(self) -> &'static str {
24        match self {
25            Self::Auto => "auto",
26            Self::On => "on",
27            Self::Off => "off",
28        }
29    }
30    pub fn parse(value: &str) -> Option<Self> {
31        [Self::Auto, Self::On, Self::Off]
32            .into_iter()
33            .find(|kind| kind.name() == value)
34    }
35}
36
37impl PromptReuse {
38    pub fn name(self) -> &'static str {
39        match self {
40            Self::Off => "off",
41            Self::Reuse => "reuse",
42        }
43    }
44    pub fn parse(value: &str) -> Option<Self> {
45        [Self::Off, Self::Reuse]
46            .into_iter()
47            .find(|kind| kind.name() == value)
48    }
49}
50
51/// Speculative decoding: `draft` uses the catalog drafter, `ngram` drafts from the
52/// context itself, and `auto` resolves to `draft` or `off` before launch.
53#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
54#[serde(rename_all = "lowercase")]
55pub enum Speculative {
56    #[default]
57    Off,
58    Auto,
59    Draft,
60    Ngram,
61}
62
63impl Speculative {
64    pub fn name(self) -> &'static str {
65        match self {
66            Self::Off => "off",
67            Self::Auto => "auto",
68            Self::Draft => "draft",
69            Self::Ngram => "ngram",
70        }
71    }
72    pub fn parse(value: &str) -> Option<Self> {
73        [Self::Off, Self::Auto, Self::Draft, Self::Ngram]
74            .into_iter()
75            .find(|kind| kind.name() == value)
76    }
77    pub fn is_off(&self) -> bool {
78        *self == Self::Off
79    }
80}
81
82pub const MAX_PARALLEL: u8 = 16;
83
84#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
85#[serde(default, rename_all = "camelCase", deny_unknown_fields)]
86pub struct CacheProfile {
87    pub kv_k: KvCacheType,
88    pub kv_v: KvCacheType,
89    pub flash_attention: FlashAttention,
90    pub prompt_reuse: PromptReuse,
91    #[serde(skip_serializing_if = "Speculative::is_off")]
92    pub speculative: Speculative,
93    /// Concurrent request slots; `None` keeps the backend default.
94    #[serde(skip_serializing_if = "Option::is_none")]
95    pub parallel: Option<u8>,
96}
97
98impl CacheProfile {
99    /// Rules that hold on every backend.
100    pub fn validate(&self) -> Result<(), CacheError> {
101        if self.kv_v != KvCacheType::F16 && self.flash_attention == FlashAttention::Off {
102            return Err(CacheError::Invalid(
103                "a quantized V cache requires flash attention",
104            ));
105        }
106        if self
107            .parallel
108            .is_some_and(|slots| !(1..=MAX_PARALLEL).contains(&slots))
109        {
110            return Err(CacheError::Invalid("parallel slots must be 1 to 16"));
111        }
112        Ok(())
113    }
114    pub fn summary(&self) -> String {
115        let kv = if self.kv_k == self.kv_v {
116            self.kv_k.name().to_owned()
117        } else {
118            format!("K {} / V {}", self.kv_k.name(), self.kv_v.name())
119        };
120        let mut text = format!(
121            "KV {kv}, flash attention {}, prompt reuse {}",
122            self.flash_attention.name(),
123            self.prompt_reuse.name()
124        );
125        if !self.speculative.is_off() {
126            text.push_str(&format!(", speculative {}", self.speculative.name()));
127        }
128        if let Some(slots) = self.parallel {
129            text.push_str(&format!(", {slots} slots"));
130        }
131        text
132    }
133}
134
135/// Cache options a user asked for; unset fields inherit from the base profile.
136#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
137pub struct CacheFlags {
138    pub kv: Option<KvCacheType>,
139    pub flash_attention: Option<FlashAttention>,
140    pub prompt_reuse: Option<PromptReuse>,
141    pub speculative: Option<Speculative>,
142    pub parallel: Option<u8>,
143}
144
145impl CacheFlags {
146    pub fn is_empty(&self) -> bool {
147        *self == Self::default()
148    }
149    /// `base` with every given flag applied on top.
150    pub fn over(self, base: Option<CacheProfile>) -> Option<CacheProfile> {
151        if self.is_empty() {
152            return base;
153        }
154        let mut profile = base.unwrap_or_default();
155        if let Some(kind) = self.kv {
156            profile.kv_k = kind;
157            profile.kv_v = kind;
158        }
159        if let Some(flash) = self.flash_attention {
160            profile.flash_attention = flash;
161        }
162        if let Some(reuse) = self.prompt_reuse {
163            profile.prompt_reuse = reuse;
164        }
165        if let Some(speculative) = self.speculative {
166            profile.speculative = speculative;
167        }
168        if let Some(slots) = self.parallel {
169            profile.parallel = Some(slots);
170        }
171        Some(profile)
172    }
173}
174
175/// Arguments and environment a cache provider adds to a backend launch.
176#[derive(Debug, Clone, Default, PartialEq, Eq)]
177pub struct SpawnDelta {
178    pub args: Vec<String>,
179    pub env: BTreeMap<String, String>,
180}
181
182#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
183pub enum CacheError {
184    #[error("invalid cache profile: {0}")]
185    Invalid(&'static str),
186    #[error("{backend} cannot apply this cache profile: {reason}")]
187    Unsupported {
188        backend: &'static str,
189        reason: &'static str,
190    },
191    #[error("{backend} cache settings are not verified yet; use the default cache profile")]
192    Unverified { backend: &'static str },
193}
194
195pub trait CacheProvider: Send + Sync {
196    fn name(&self) -> &'static str;
197    fn apply(
198        &self,
199        profile: &CacheProfile,
200        backend: &str,
201        owned: bool,
202    ) -> Result<SpawnDelta, CacheError>;
203}
204
205/// Providers are compiled in; names never load code or pass raw arguments through.
206pub fn provider(name: &str) -> Option<&'static dyn CacheProvider> {
207    match name {
208        "native" => Some(&NativeCacheProvider),
209        _ => None,
210    }
211}
212
213/// The profile worth recording and applying: `None` when absent or equal to the default.
214pub fn effective(profile: Option<&CacheProfile>) -> Option<CacheProfile> {
215    profile
216        .copied()
217        .filter(|profile| *profile != CacheProfile::default())
218}
219
220/// The profile a launch applies: flags win, then the running profile for `switch`
221/// or the user default for `up`, then the backend default (`None`).
222pub fn requested(
223    flags: CacheFlags,
224    switch: bool,
225    running: Option<CacheProfile>,
226    user_default: Option<CacheProfile>,
227) -> Option<CacheProfile> {
228    effective(
229        flags
230            .over(if switch { running } else { user_default })
231            .as_ref(),
232    )
233}
234
235/// Launch settings for `backend`, refusing profiles it cannot apply before anything starts.
236pub fn launch_delta(
237    profile: Option<&CacheProfile>,
238    backend: &str,
239    owned: bool,
240) -> Result<SpawnDelta, CacheError> {
241    match effective(profile) {
242        None => Ok(SpawnDelta::default()),
243        Some(profile) => NativeCacheProvider.apply(&profile, backend, owned),
244    }
245}
246
247/// Cache settings each backend already exposes (verified: llama.cpp b10090, Ollama 0.32.5).
248pub struct NativeCacheProvider;
249
250impl CacheProvider for NativeCacheProvider {
251    fn name(&self) -> &'static str {
252        "native"
253    }
254    fn apply(
255        &self,
256        profile: &CacheProfile,
257        backend: &str,
258        owned: bool,
259    ) -> Result<SpawnDelta, CacheError> {
260        if !rigspark_core::catalog::BACKENDS.contains(&backend) {
261            return Err(CacheError::Invalid("unknown backend"));
262        }
263        profile.validate()?;
264        // An unresolved `auto` asks for nothing by itself.
265        let profile = &CacheProfile {
266            speculative: match profile.speculative {
267                Speculative::Auto => Speculative::Off,
268                mode => mode,
269            },
270            ..*profile
271        };
272        if *profile == CacheProfile::default() {
273            return Ok(SpawnDelta::default());
274        }
275        match backend {
276            "llamacpp" => Ok(llama_cpp(profile)),
277            "ollama" => ollama(profile, owned),
278            "mlx" => Err(CacheError::Unverified { backend: "mlx" }),
279            "bitnet" => Err(CacheError::Unverified { backend: "bitnet" }),
280            _ => Err(CacheError::Unsupported {
281                backend: "lmstudio",
282                reason: "LM Studio is attach-only; change its cache settings in LM Studio",
283            }),
284        }
285    }
286}
287
288fn flash_enabled(profile: &CacheProfile) -> Option<bool> {
289    match profile.flash_attention {
290        FlashAttention::On => Some(true),
291        FlashAttention::Off => Some(false),
292        FlashAttention::Auto => (profile.kv_v != KvCacheType::F16).then_some(true),
293    }
294}
295
296fn llama_cpp(profile: &CacheProfile) -> SpawnDelta {
297    let mut args = Vec::new();
298    for (flag, kind) in [
299        ("--cache-type-k", profile.kv_k),
300        ("--cache-type-v", profile.kv_v),
301    ] {
302        if kind != KvCacheType::F16 {
303            args.extend([flag.to_string(), kind.name().to_string()]);
304        }
305    }
306    if let Some(enabled) = flash_enabled(profile) {
307        args.extend([
308            "--flash-attn".into(),
309            if enabled { "on" } else { "off" }.into(),
310        ]);
311    }
312    if profile.prompt_reuse == PromptReuse::Reuse {
313        args.extend(["--cache-reuse".into(), "256".into()]);
314    }
315    match profile.speculative {
316        Speculative::Draft => args.extend(["--spec-type".into(), "draft-simple".into()]),
317        Speculative::Ngram => args.extend(["--spec-type".into(), "ngram-mod".into()]),
318        Speculative::Off | Speculative::Auto => {}
319    }
320    if let Some(slots) = profile.parallel {
321        args.extend([
322            "--parallel".into(),
323            slots.to_string(),
324            "--kv-unified".into(),
325        ]);
326    }
327    SpawnDelta {
328        args,
329        env: BTreeMap::new(),
330    }
331}
332
333fn ollama(profile: &CacheProfile, owned: bool) -> Result<SpawnDelta, CacheError> {
334    let unsupported = |reason| CacheError::Unsupported {
335        backend: "ollama",
336        reason,
337    };
338    if profile.kv_k != profile.kv_v {
339        return Err(unsupported("Ollama uses one cache type for both K and V"));
340    }
341    let mut env = BTreeMap::new();
342    if let Some(enabled) = flash_enabled(profile) {
343        env.insert(
344            "OLLAMA_FLASH_ATTENTION".into(),
345            if enabled { "1" } else { "0" }.into(),
346        );
347    }
348    if profile.kv_k != KvCacheType::F16 {
349        env.insert("OLLAMA_KV_CACHE_TYPE".into(), profile.kv_k.name().into());
350    }
351    if matches!(profile.speculative, Speculative::Draft | Speculative::Ngram) {
352        return Err(unsupported(
353            "Ollama ships speculative decoding inside its model tags; use llama.cpp to choose a drafter",
354        ));
355    }
356    if let Some(slots) = profile.parallel {
357        env.insert("OLLAMA_NUM_PARALLEL".into(), slots.to_string());
358    }
359    // Prompt-prefix reuse is Ollama's default behaviour and needs no setting.
360    if !env.is_empty() && !owned {
361        return Err(unsupported(
362            "cache settings are daemon-wide and apply only to an Ollama daemon rigspark starts",
363        ));
364    }
365    Ok(SpawnDelta {
366        args: Vec::new(),
367        env,
368    })
369}