Skip to main content

rigspark_runtime/
cache.rs

1use rigspark_core::sizing::KvCacheType;
2use serde::{Deserialize, Serialize};
3use std::collections::BTreeMap;
4
5#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
6#[serde(rename_all = "lowercase")]
7pub enum FlashAttention {
8    #[default]
9    Auto,
10    On,
11    Off,
12}
13
14#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
15#[serde(rename_all = "lowercase")]
16pub enum PromptReuse {
17    #[default]
18    Off,
19    Reuse,
20}
21
22impl FlashAttention {
23    pub fn name(self) -> &'static str {
24        match self {
25            Self::Auto => "auto",
26            Self::On => "on",
27            Self::Off => "off",
28        }
29    }
30    pub fn parse(value: &str) -> Option<Self> {
31        [Self::Auto, Self::On, Self::Off]
32            .into_iter()
33            .find(|kind| kind.name() == value)
34    }
35}
36
37impl PromptReuse {
38    pub fn name(self) -> &'static str {
39        match self {
40            Self::Off => "off",
41            Self::Reuse => "reuse",
42        }
43    }
44    pub fn parse(value: &str) -> Option<Self> {
45        [Self::Off, Self::Reuse]
46            .into_iter()
47            .find(|kind| kind.name() == value)
48    }
49}
50
51#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
52#[serde(default, rename_all = "camelCase", deny_unknown_fields)]
53pub struct CacheProfile {
54    pub kv_k: KvCacheType,
55    pub kv_v: KvCacheType,
56    pub flash_attention: FlashAttention,
57    pub prompt_reuse: PromptReuse,
58}
59
60impl CacheProfile {
61    /// Rules that hold on every backend.
62    pub fn validate(&self) -> Result<(), CacheError> {
63        if self.kv_v != KvCacheType::F16 && self.flash_attention == FlashAttention::Off {
64            return Err(CacheError::Invalid(
65                "a quantized V cache requires flash attention",
66            ));
67        }
68        Ok(())
69    }
70    pub fn summary(&self) -> String {
71        let kv = if self.kv_k == self.kv_v {
72            self.kv_k.name().to_owned()
73        } else {
74            format!("K {} / V {}", self.kv_k.name(), self.kv_v.name())
75        };
76        format!(
77            "KV {kv}, flash attention {}, prompt reuse {}",
78            self.flash_attention.name(),
79            self.prompt_reuse.name()
80        )
81    }
82}
83
84/// Cache options a user asked for; unset fields inherit from the base profile.
85#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
86pub struct CacheFlags {
87    pub kv: Option<KvCacheType>,
88    pub flash_attention: Option<FlashAttention>,
89    pub prompt_reuse: Option<PromptReuse>,
90}
91
92impl CacheFlags {
93    pub fn is_empty(&self) -> bool {
94        *self == Self::default()
95    }
96    /// `base` with every given flag applied on top.
97    pub fn over(self, base: Option<CacheProfile>) -> Option<CacheProfile> {
98        if self.is_empty() {
99            return base;
100        }
101        let mut profile = base.unwrap_or_default();
102        if let Some(kind) = self.kv {
103            profile.kv_k = kind;
104            profile.kv_v = kind;
105        }
106        if let Some(flash) = self.flash_attention {
107            profile.flash_attention = flash;
108        }
109        if let Some(reuse) = self.prompt_reuse {
110            profile.prompt_reuse = reuse;
111        }
112        Some(profile)
113    }
114}
115
116/// Arguments and environment a cache provider adds to a backend launch.
117#[derive(Debug, Clone, Default, PartialEq, Eq)]
118pub struct SpawnDelta {
119    pub args: Vec<String>,
120    pub env: BTreeMap<String, String>,
121}
122
123#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
124pub enum CacheError {
125    #[error("invalid cache profile: {0}")]
126    Invalid(&'static str),
127    #[error("{backend} cannot apply this cache profile: {reason}")]
128    Unsupported {
129        backend: &'static str,
130        reason: &'static str,
131    },
132    #[error("{backend} cache settings are not verified yet; use the default cache profile")]
133    Unverified { backend: &'static str },
134}
135
136pub trait CacheProvider: Send + Sync {
137    fn name(&self) -> &'static str;
138    fn apply(
139        &self,
140        profile: &CacheProfile,
141        backend: &str,
142        owned: bool,
143    ) -> Result<SpawnDelta, CacheError>;
144}
145
146/// Providers are compiled in; names never load code or pass raw arguments through.
147pub fn provider(name: &str) -> Option<&'static dyn CacheProvider> {
148    match name {
149        "native" => Some(&NativeCacheProvider),
150        _ => None,
151    }
152}
153
154/// The profile worth recording and applying: `None` when absent or equal to the default.
155pub fn effective(profile: Option<&CacheProfile>) -> Option<CacheProfile> {
156    profile
157        .copied()
158        .filter(|profile| *profile != CacheProfile::default())
159}
160
161/// The profile a launch applies: flags win, then the running profile for `switch`
162/// or the user default for `up`, then the backend default (`None`).
163pub fn requested(
164    flags: CacheFlags,
165    switch: bool,
166    running: Option<CacheProfile>,
167    user_default: Option<CacheProfile>,
168) -> Option<CacheProfile> {
169    effective(
170        flags
171            .over(if switch { running } else { user_default })
172            .as_ref(),
173    )
174}
175
176/// Launch settings for `backend`, refusing profiles it cannot apply before anything starts.
177pub fn launch_delta(
178    profile: Option<&CacheProfile>,
179    backend: &str,
180    owned: bool,
181) -> Result<SpawnDelta, CacheError> {
182    match effective(profile) {
183        None => Ok(SpawnDelta::default()),
184        Some(profile) => NativeCacheProvider.apply(&profile, backend, owned),
185    }
186}
187
188/// Cache settings each backend already exposes (verified: llama.cpp b10090, Ollama 0.32.5).
189pub struct NativeCacheProvider;
190
191impl CacheProvider for NativeCacheProvider {
192    fn name(&self) -> &'static str {
193        "native"
194    }
195    fn apply(
196        &self,
197        profile: &CacheProfile,
198        backend: &str,
199        owned: bool,
200    ) -> Result<SpawnDelta, CacheError> {
201        if !rigspark_core::catalog::BACKENDS.contains(&backend) {
202            return Err(CacheError::Invalid("unknown backend"));
203        }
204        profile.validate()?;
205        if *profile == CacheProfile::default() {
206            return Ok(SpawnDelta::default());
207        }
208        match backend {
209            "llamacpp" => Ok(llama_cpp(profile)),
210            "ollama" => ollama(profile, owned),
211            "mlx" => Err(CacheError::Unverified { backend: "mlx" }),
212            "bitnet" => Err(CacheError::Unverified { backend: "bitnet" }),
213            _ => Err(CacheError::Unsupported {
214                backend: "lmstudio",
215                reason: "LM Studio is attach-only; change its cache settings in LM Studio",
216            }),
217        }
218    }
219}
220
221fn flash_enabled(profile: &CacheProfile) -> Option<bool> {
222    match profile.flash_attention {
223        FlashAttention::On => Some(true),
224        FlashAttention::Off => Some(false),
225        FlashAttention::Auto => (profile.kv_v != KvCacheType::F16).then_some(true),
226    }
227}
228
229fn llama_cpp(profile: &CacheProfile) -> SpawnDelta {
230    let mut args = Vec::new();
231    for (flag, kind) in [
232        ("--cache-type-k", profile.kv_k),
233        ("--cache-type-v", profile.kv_v),
234    ] {
235        if kind != KvCacheType::F16 {
236            args.extend([flag.to_string(), kind.name().to_string()]);
237        }
238    }
239    if let Some(enabled) = flash_enabled(profile) {
240        args.extend([
241            "--flash-attn".into(),
242            if enabled { "on" } else { "off" }.into(),
243        ]);
244    }
245    if profile.prompt_reuse == PromptReuse::Reuse {
246        args.extend(["--cache-reuse".into(), "256".into()]);
247    }
248    SpawnDelta {
249        args,
250        env: BTreeMap::new(),
251    }
252}
253
254fn ollama(profile: &CacheProfile, owned: bool) -> Result<SpawnDelta, CacheError> {
255    let unsupported = |reason| CacheError::Unsupported {
256        backend: "ollama",
257        reason,
258    };
259    if profile.kv_k != profile.kv_v {
260        return Err(unsupported("Ollama uses one cache type for both K and V"));
261    }
262    let mut env = BTreeMap::new();
263    if let Some(enabled) = flash_enabled(profile) {
264        env.insert(
265            "OLLAMA_FLASH_ATTENTION".into(),
266            if enabled { "1" } else { "0" }.into(),
267        );
268    }
269    if profile.kv_k != KvCacheType::F16 {
270        env.insert("OLLAMA_KV_CACHE_TYPE".into(), profile.kv_k.name().into());
271    }
272    // Prompt-prefix reuse is Ollama's default behaviour and needs no setting.
273    if !env.is_empty() && !owned {
274        return Err(unsupported(
275            "cache settings are daemon-wide and apply only to an Ollama daemon rigspark starts",
276        ));
277    }
278    Ok(SpawnDelta {
279        args: Vec::new(),
280        env,
281    })
282}