1use rigspark_core::sizing::KvCacheType;
2use serde::{Deserialize, Serialize};
3use std::collections::BTreeMap;
4
5#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
6#[serde(rename_all = "lowercase")]
7pub enum FlashAttention {
8 #[default]
9 Auto,
10 On,
11 Off,
12}
13
14#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
15#[serde(rename_all = "lowercase")]
16pub enum PromptReuse {
17 #[default]
18 Off,
19 Reuse,
20}
21
22impl FlashAttention {
23 pub fn name(self) -> &'static str {
24 match self {
25 Self::Auto => "auto",
26 Self::On => "on",
27 Self::Off => "off",
28 }
29 }
30 pub fn parse(value: &str) -> Option<Self> {
31 [Self::Auto, Self::On, Self::Off]
32 .into_iter()
33 .find(|kind| kind.name() == value)
34 }
35}
36
37impl PromptReuse {
38 pub fn name(self) -> &'static str {
39 match self {
40 Self::Off => "off",
41 Self::Reuse => "reuse",
42 }
43 }
44 pub fn parse(value: &str) -> Option<Self> {
45 [Self::Off, Self::Reuse]
46 .into_iter()
47 .find(|kind| kind.name() == value)
48 }
49}
50
51#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
54#[serde(rename_all = "lowercase")]
55pub enum Speculative {
56 #[default]
57 Off,
58 Auto,
59 Draft,
60 Ngram,
61}
62
63impl Speculative {
64 pub fn name(self) -> &'static str {
65 match self {
66 Self::Off => "off",
67 Self::Auto => "auto",
68 Self::Draft => "draft",
69 Self::Ngram => "ngram",
70 }
71 }
72 pub fn parse(value: &str) -> Option<Self> {
73 [Self::Off, Self::Auto, Self::Draft, Self::Ngram]
74 .into_iter()
75 .find(|kind| kind.name() == value)
76 }
77 pub fn is_off(&self) -> bool {
78 *self == Self::Off
79 }
80}
81
82pub const MAX_PARALLEL: u8 = 16;
83
84#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
85#[serde(default, rename_all = "camelCase", deny_unknown_fields)]
86pub struct CacheProfile {
87 pub kv_k: KvCacheType,
88 pub kv_v: KvCacheType,
89 pub flash_attention: FlashAttention,
90 pub prompt_reuse: PromptReuse,
91 #[serde(skip_serializing_if = "Speculative::is_off")]
92 pub speculative: Speculative,
93 #[serde(skip_serializing_if = "Option::is_none")]
95 pub parallel: Option<u8>,
96}
97
98impl CacheProfile {
99 pub fn validate(&self) -> Result<(), CacheError> {
101 if self.kv_v != KvCacheType::F16 && self.flash_attention == FlashAttention::Off {
102 return Err(CacheError::Invalid(
103 "a quantized V cache requires flash attention",
104 ));
105 }
106 if self
107 .parallel
108 .is_some_and(|slots| !(1..=MAX_PARALLEL).contains(&slots))
109 {
110 return Err(CacheError::Invalid("parallel slots must be 1 to 16"));
111 }
112 Ok(())
113 }
114 pub fn summary(&self) -> String {
115 let kv = if self.kv_k == self.kv_v {
116 self.kv_k.name().to_owned()
117 } else {
118 format!("K {} / V {}", self.kv_k.name(), self.kv_v.name())
119 };
120 let mut text = format!(
121 "KV {kv}, flash attention {}, prompt reuse {}",
122 self.flash_attention.name(),
123 self.prompt_reuse.name()
124 );
125 if !self.speculative.is_off() {
126 text.push_str(&format!(", speculative {}", self.speculative.name()));
127 }
128 if let Some(slots) = self.parallel {
129 text.push_str(&format!(", {slots} slots"));
130 }
131 text
132 }
133}
134
135#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
137pub struct CacheFlags {
138 pub kv: Option<KvCacheType>,
139 pub flash_attention: Option<FlashAttention>,
140 pub prompt_reuse: Option<PromptReuse>,
141 pub speculative: Option<Speculative>,
142 pub parallel: Option<u8>,
143}
144
145impl CacheFlags {
146 pub fn is_empty(&self) -> bool {
147 *self == Self::default()
148 }
149 pub fn over(self, base: Option<CacheProfile>) -> Option<CacheProfile> {
151 if self.is_empty() {
152 return base;
153 }
154 let mut profile = base.unwrap_or_default();
155 if let Some(kind) = self.kv {
156 profile.kv_k = kind;
157 profile.kv_v = kind;
158 }
159 if let Some(flash) = self.flash_attention {
160 profile.flash_attention = flash;
161 }
162 if let Some(reuse) = self.prompt_reuse {
163 profile.prompt_reuse = reuse;
164 }
165 if let Some(speculative) = self.speculative {
166 profile.speculative = speculative;
167 }
168 if let Some(slots) = self.parallel {
169 profile.parallel = Some(slots);
170 }
171 Some(profile)
172 }
173}
174
175#[derive(Debug, Clone, Default, PartialEq, Eq)]
177pub struct SpawnDelta {
178 pub args: Vec<String>,
179 pub env: BTreeMap<String, String>,
180}
181
182#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
183pub enum CacheError {
184 #[error("invalid cache profile: {0}")]
185 Invalid(&'static str),
186 #[error("{backend} cannot apply this cache profile: {reason}")]
187 Unsupported {
188 backend: &'static str,
189 reason: &'static str,
190 },
191 #[error("{backend} cache settings are not verified yet; use the default cache profile")]
192 Unverified { backend: &'static str },
193}
194
195pub trait CacheProvider: Send + Sync {
196 fn name(&self) -> &'static str;
197 fn apply(
198 &self,
199 profile: &CacheProfile,
200 backend: &str,
201 owned: bool,
202 ) -> Result<SpawnDelta, CacheError>;
203}
204
205pub fn provider(name: &str) -> Option<&'static dyn CacheProvider> {
207 match name {
208 "native" => Some(&NativeCacheProvider),
209 _ => None,
210 }
211}
212
213pub fn effective(profile: Option<&CacheProfile>) -> Option<CacheProfile> {
215 profile
216 .copied()
217 .filter(|profile| *profile != CacheProfile::default())
218}
219
220pub fn requested(
223 flags: CacheFlags,
224 switch: bool,
225 running: Option<CacheProfile>,
226 user_default: Option<CacheProfile>,
227) -> Option<CacheProfile> {
228 effective(
229 flags
230 .over(if switch { running } else { user_default })
231 .as_ref(),
232 )
233}
234
235pub fn launch_delta(
237 profile: Option<&CacheProfile>,
238 backend: &str,
239 owned: bool,
240) -> Result<SpawnDelta, CacheError> {
241 match effective(profile) {
242 None => Ok(SpawnDelta::default()),
243 Some(profile) => NativeCacheProvider.apply(&profile, backend, owned),
244 }
245}
246
247pub struct NativeCacheProvider;
249
250impl CacheProvider for NativeCacheProvider {
251 fn name(&self) -> &'static str {
252 "native"
253 }
254 fn apply(
255 &self,
256 profile: &CacheProfile,
257 backend: &str,
258 owned: bool,
259 ) -> Result<SpawnDelta, CacheError> {
260 if !rigspark_core::catalog::BACKENDS.contains(&backend) {
261 return Err(CacheError::Invalid("unknown backend"));
262 }
263 profile.validate()?;
264 let profile = &CacheProfile {
266 speculative: match profile.speculative {
267 Speculative::Auto => Speculative::Off,
268 mode => mode,
269 },
270 ..*profile
271 };
272 if *profile == CacheProfile::default() {
273 return Ok(SpawnDelta::default());
274 }
275 match backend {
276 "llamacpp" => Ok(llama_cpp(profile)),
277 "ollama" => ollama(profile, owned),
278 "mlx" => Err(CacheError::Unverified { backend: "mlx" }),
279 "bitnet" => Err(CacheError::Unverified { backend: "bitnet" }),
280 _ => Err(CacheError::Unsupported {
281 backend: "lmstudio",
282 reason: "LM Studio is attach-only; change its cache settings in LM Studio",
283 }),
284 }
285 }
286}
287
288fn flash_enabled(profile: &CacheProfile) -> Option<bool> {
289 match profile.flash_attention {
290 FlashAttention::On => Some(true),
291 FlashAttention::Off => Some(false),
292 FlashAttention::Auto => (profile.kv_v != KvCacheType::F16).then_some(true),
293 }
294}
295
296fn llama_cpp(profile: &CacheProfile) -> SpawnDelta {
297 let mut args = Vec::new();
298 for (flag, kind) in [
299 ("--cache-type-k", profile.kv_k),
300 ("--cache-type-v", profile.kv_v),
301 ] {
302 if kind != KvCacheType::F16 {
303 args.extend([flag.to_string(), kind.name().to_string()]);
304 }
305 }
306 if let Some(enabled) = flash_enabled(profile) {
307 args.extend([
308 "--flash-attn".into(),
309 if enabled { "on" } else { "off" }.into(),
310 ]);
311 }
312 if profile.prompt_reuse == PromptReuse::Reuse {
313 args.extend(["--cache-reuse".into(), "256".into()]);
314 }
315 match profile.speculative {
316 Speculative::Draft => args.extend(["--spec-type".into(), "draft-simple".into()]),
317 Speculative::Ngram => args.extend(["--spec-type".into(), "ngram-mod".into()]),
318 Speculative::Off | Speculative::Auto => {}
319 }
320 if let Some(slots) = profile.parallel {
321 args.extend([
322 "--parallel".into(),
323 slots.to_string(),
324 "--kv-unified".into(),
325 ]);
326 }
327 SpawnDelta {
328 args,
329 env: BTreeMap::new(),
330 }
331}
332
333fn ollama(profile: &CacheProfile, owned: bool) -> Result<SpawnDelta, CacheError> {
334 let unsupported = |reason| CacheError::Unsupported {
335 backend: "ollama",
336 reason,
337 };
338 if profile.kv_k != profile.kv_v {
339 return Err(unsupported("Ollama uses one cache type for both K and V"));
340 }
341 let mut env = BTreeMap::new();
342 if let Some(enabled) = flash_enabled(profile) {
343 env.insert(
344 "OLLAMA_FLASH_ATTENTION".into(),
345 if enabled { "1" } else { "0" }.into(),
346 );
347 }
348 if profile.kv_k != KvCacheType::F16 {
349 env.insert("OLLAMA_KV_CACHE_TYPE".into(), profile.kv_k.name().into());
350 }
351 if matches!(profile.speculative, Speculative::Draft | Speculative::Ngram) {
352 return Err(unsupported(
353 "Ollama ships speculative decoding inside its model tags; use llama.cpp to choose a drafter",
354 ));
355 }
356 if let Some(slots) = profile.parallel {
357 env.insert("OLLAMA_NUM_PARALLEL".into(), slots.to_string());
358 }
359 if !env.is_empty() && !owned {
361 return Err(unsupported(
362 "cache settings are daemon-wide and apply only to an Ollama daemon rigspark starts",
363 ));
364 }
365 Ok(SpawnDelta {
366 args: Vec::new(),
367 env,
368 })
369}