1use rigspark_core::sizing::KvCacheType;
2use serde::{Deserialize, Serialize};
3use std::collections::BTreeMap;
4
5#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
6#[serde(rename_all = "lowercase")]
7pub enum FlashAttention {
8 #[default]
9 Auto,
10 On,
11 Off,
12}
13
14#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
15#[serde(rename_all = "lowercase")]
16pub enum PromptReuse {
17 #[default]
18 Off,
19 Reuse,
20}
21
22impl FlashAttention {
23 pub fn name(self) -> &'static str {
24 match self {
25 Self::Auto => "auto",
26 Self::On => "on",
27 Self::Off => "off",
28 }
29 }
30 pub fn parse(value: &str) -> Option<Self> {
31 [Self::Auto, Self::On, Self::Off]
32 .into_iter()
33 .find(|kind| kind.name() == value)
34 }
35}
36
37impl PromptReuse {
38 pub fn name(self) -> &'static str {
39 match self {
40 Self::Off => "off",
41 Self::Reuse => "reuse",
42 }
43 }
44 pub fn parse(value: &str) -> Option<Self> {
45 [Self::Off, Self::Reuse]
46 .into_iter()
47 .find(|kind| kind.name() == value)
48 }
49}
50
51#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
52#[serde(default, rename_all = "camelCase", deny_unknown_fields)]
53pub struct CacheProfile {
54 pub kv_k: KvCacheType,
55 pub kv_v: KvCacheType,
56 pub flash_attention: FlashAttention,
57 pub prompt_reuse: PromptReuse,
58}
59
60impl CacheProfile {
61 pub fn validate(&self) -> Result<(), CacheError> {
63 if self.kv_v != KvCacheType::F16 && self.flash_attention == FlashAttention::Off {
64 return Err(CacheError::Invalid(
65 "a quantized V cache requires flash attention",
66 ));
67 }
68 Ok(())
69 }
70 pub fn summary(&self) -> String {
71 let kv = if self.kv_k == self.kv_v {
72 self.kv_k.name().to_owned()
73 } else {
74 format!("K {} / V {}", self.kv_k.name(), self.kv_v.name())
75 };
76 format!(
77 "KV {kv}, flash attention {}, prompt reuse {}",
78 self.flash_attention.name(),
79 self.prompt_reuse.name()
80 )
81 }
82}
83
84#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
86pub struct CacheFlags {
87 pub kv: Option<KvCacheType>,
88 pub flash_attention: Option<FlashAttention>,
89 pub prompt_reuse: Option<PromptReuse>,
90}
91
92impl CacheFlags {
93 pub fn is_empty(&self) -> bool {
94 *self == Self::default()
95 }
96 pub fn over(self, base: Option<CacheProfile>) -> Option<CacheProfile> {
98 if self.is_empty() {
99 return base;
100 }
101 let mut profile = base.unwrap_or_default();
102 if let Some(kind) = self.kv {
103 profile.kv_k = kind;
104 profile.kv_v = kind;
105 }
106 if let Some(flash) = self.flash_attention {
107 profile.flash_attention = flash;
108 }
109 if let Some(reuse) = self.prompt_reuse {
110 profile.prompt_reuse = reuse;
111 }
112 Some(profile)
113 }
114}
115
116#[derive(Debug, Clone, Default, PartialEq, Eq)]
118pub struct SpawnDelta {
119 pub args: Vec<String>,
120 pub env: BTreeMap<String, String>,
121}
122
123#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
124pub enum CacheError {
125 #[error("invalid cache profile: {0}")]
126 Invalid(&'static str),
127 #[error("{backend} cannot apply this cache profile: {reason}")]
128 Unsupported {
129 backend: &'static str,
130 reason: &'static str,
131 },
132 #[error("{backend} cache settings are not verified yet; use the default cache profile")]
133 Unverified { backend: &'static str },
134}
135
136pub trait CacheProvider: Send + Sync {
137 fn name(&self) -> &'static str;
138 fn apply(
139 &self,
140 profile: &CacheProfile,
141 backend: &str,
142 owned: bool,
143 ) -> Result<SpawnDelta, CacheError>;
144}
145
146pub fn provider(name: &str) -> Option<&'static dyn CacheProvider> {
148 match name {
149 "native" => Some(&NativeCacheProvider),
150 _ => None,
151 }
152}
153
154pub fn effective(profile: Option<&CacheProfile>) -> Option<CacheProfile> {
156 profile
157 .copied()
158 .filter(|profile| *profile != CacheProfile::default())
159}
160
161pub fn requested(
164 flags: CacheFlags,
165 switch: bool,
166 running: Option<CacheProfile>,
167 user_default: Option<CacheProfile>,
168) -> Option<CacheProfile> {
169 effective(
170 flags
171 .over(if switch { running } else { user_default })
172 .as_ref(),
173 )
174}
175
176pub fn launch_delta(
178 profile: Option<&CacheProfile>,
179 backend: &str,
180 owned: bool,
181) -> Result<SpawnDelta, CacheError> {
182 match effective(profile) {
183 None => Ok(SpawnDelta::default()),
184 Some(profile) => NativeCacheProvider.apply(&profile, backend, owned),
185 }
186}
187
188pub struct NativeCacheProvider;
190
191impl CacheProvider for NativeCacheProvider {
192 fn name(&self) -> &'static str {
193 "native"
194 }
195 fn apply(
196 &self,
197 profile: &CacheProfile,
198 backend: &str,
199 owned: bool,
200 ) -> Result<SpawnDelta, CacheError> {
201 if !rigspark_core::catalog::BACKENDS.contains(&backend) {
202 return Err(CacheError::Invalid("unknown backend"));
203 }
204 profile.validate()?;
205 if *profile == CacheProfile::default() {
206 return Ok(SpawnDelta::default());
207 }
208 match backend {
209 "llamacpp" => Ok(llama_cpp(profile)),
210 "ollama" => ollama(profile, owned),
211 "mlx" => Err(CacheError::Unverified { backend: "mlx" }),
212 _ => Err(CacheError::Unsupported {
213 backend: "lmstudio",
214 reason: "LM Studio is attach-only; change its cache settings in LM Studio",
215 }),
216 }
217 }
218}
219
220fn flash_enabled(profile: &CacheProfile) -> Option<bool> {
221 match profile.flash_attention {
222 FlashAttention::On => Some(true),
223 FlashAttention::Off => Some(false),
224 FlashAttention::Auto => (profile.kv_v != KvCacheType::F16).then_some(true),
225 }
226}
227
228fn llama_cpp(profile: &CacheProfile) -> SpawnDelta {
229 let mut args = Vec::new();
230 for (flag, kind) in [
231 ("--cache-type-k", profile.kv_k),
232 ("--cache-type-v", profile.kv_v),
233 ] {
234 if kind != KvCacheType::F16 {
235 args.extend([flag.to_string(), kind.name().to_string()]);
236 }
237 }
238 if let Some(enabled) = flash_enabled(profile) {
239 args.extend([
240 "--flash-attn".into(),
241 if enabled { "on" } else { "off" }.into(),
242 ]);
243 }
244 if profile.prompt_reuse == PromptReuse::Reuse {
245 args.extend(["--cache-reuse".into(), "256".into()]);
246 }
247 SpawnDelta {
248 args,
249 env: BTreeMap::new(),
250 }
251}
252
253fn ollama(profile: &CacheProfile, owned: bool) -> Result<SpawnDelta, CacheError> {
254 let unsupported = |reason| CacheError::Unsupported {
255 backend: "ollama",
256 reason,
257 };
258 if profile.kv_k != profile.kv_v {
259 return Err(unsupported("Ollama uses one cache type for both K and V"));
260 }
261 let mut env = BTreeMap::new();
262 if let Some(enabled) = flash_enabled(profile) {
263 env.insert(
264 "OLLAMA_FLASH_ATTENTION".into(),
265 if enabled { "1" } else { "0" }.into(),
266 );
267 }
268 if profile.kv_k != KvCacheType::F16 {
269 env.insert("OLLAMA_KV_CACHE_TYPE".into(), profile.kv_k.name().into());
270 }
271 if !env.is_empty() && !owned {
273 return Err(unsupported(
274 "cache settings are daemon-wide and apply only to an Ollama daemon rigspark starts",
275 ));
276 }
277 Ok(SpawnDelta {
278 args: Vec::new(),
279 env,
280 })
281}