Skip to main content

reflex/pulse/write/
cache.rs

1//! Content-addressed cache for the Pulse LLM writing pass.
2//!
3//! The key is the hash of exactly what would be sent: task kind, prompt version,
4//! provider, model, output contract, and the full system and user text. There is no
5//! snapshot id and no wall clock in it, so an unchanged request hits the cache after
6//! any number of re-indexes, on any machine.
7//!
8//! Layout:
9//! ```text
10//! <dir>/v1/ab/abcdef….json   one entry per key; no timestamps, stable bytes
11//! <dir>/runs.json            keys referenced by the last K successful runs
12//! ```
13//! Entries are safe to commit or to keep in a CI cache: any older cache is useful.
14
15use crate::atomic_write::{atomic_replace, tmp_path_for};
16use crate::semantic::providers::Usage;
17use anyhow::{Context, Result};
18use serde::{Deserialize, Serialize};
19use std::collections::BTreeSet;
20use std::path::{Path, PathBuf};
21
22const LAYOUT: &str = "v1";
23const KEY_SCHEMA: &str = "pulse-write/1";
24
25/// Everything that decides the model's answer. Serialized to JSON and hashed.
26#[derive(Serialize)]
27pub struct KeyMaterial<'a> {
28    pub task_kind: &'a str,
29    pub prompt_version: u32,
30    /// `None` when `cache_model_agnostic` is set.
31    pub provider: Option<&'a str>,
32    pub model: Option<&'a str>,
33    pub max_tokens: u32,
34    pub output_mode: &'a str,
35    /// Canonical JSON of the output schema, if any.
36    pub output_schema: Option<&'a str>,
37    pub system: &'a str,
38    pub user: &'a str,
39}
40
41impl KeyMaterial<'_> {
42    pub fn key(&self) -> String {
43        #[derive(Serialize)]
44        struct Versioned<'a, 'b> {
45            schema: &'static str,
46            #[serde(flatten)]
47            material: &'a KeyMaterial<'b>,
48        }
49        let bytes = serde_json::to_vec(&Versioned {
50            schema: KEY_SCHEMA,
51            material: self,
52        })
53        .expect("KeyMaterial serializes");
54        blake3::hash(&bytes).to_hex().to_string()
55    }
56}
57
58/// One cached answer.
59#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
60pub struct CacheEntry {
61    pub key: String,
62    pub task_id: String,
63    pub task_kind: String,
64    pub model: String,
65    #[serde(default)]
66    pub usage: Option<Usage>,
67    pub text: String,
68}
69
70#[derive(Debug, Default, Serialize, Deserialize)]
71struct RunsFile {
72    runs: Vec<RunRecord>,
73}
74
75#[derive(Debug, Serialize, Deserialize)]
76struct RunRecord {
77    keys: Vec<String>,
78}
79
80/// The write cache rooted at one directory.
81#[derive(Debug, Clone)]
82pub struct WriteCache {
83    dir: PathBuf,
84}
85
86impl WriteCache {
87    /// Default location: `<reflex cache>/pulse/write-cache`.
88    pub fn default_dir(reflex_cache: &Path) -> PathBuf {
89        reflex_cache.join("pulse").join("write-cache")
90    }
91
92    pub fn new(dir: impl Into<PathBuf>) -> Self {
93        Self { dir: dir.into() }
94    }
95
96    pub fn dir(&self) -> &Path {
97        &self.dir
98    }
99
100    fn entry_path(&self, key: &str) -> PathBuf {
101        let shard = key.get(..2).unwrap_or("00");
102        self.dir
103            .join(LAYOUT)
104            .join(shard)
105            .join(format!("{key}.json"))
106    }
107
108    /// Read an entry. A missing or unreadable entry is a miss, never an error.
109    pub fn get(&self, key: &str) -> Option<CacheEntry> {
110        let bytes = std::fs::read(self.entry_path(key)).ok()?;
111        match serde_json::from_slice::<CacheEntry>(&bytes) {
112            Ok(entry) if entry.key == key => Some(entry),
113            Ok(_) => None,
114            Err(e) => {
115                log::warn!("ignoring corrupt pulse write-cache entry {key}: {e}");
116                None
117            }
118        }
119    }
120
121    /// Write an entry atomically (temp file + rename), so concurrent tasks never conflict.
122    pub fn put(&self, entry: &CacheEntry) -> Result<()> {
123        let path = self.entry_path(&entry.key);
124        let parent = path.parent().expect("entry path has a parent");
125        std::fs::create_dir_all(parent)
126            .with_context(|| format!("creating {}", parent.display()))?;
127        let tmp = tmp_path_for(&path);
128        std::fs::write(&tmp, serde_json::to_vec_pretty(entry)?)
129            .with_context(|| format!("writing {}", tmp.display()))?;
130        atomic_replace(&tmp, &path)?;
131        Ok(())
132    }
133
134    /// Record the keys one successful run used, keep the last `keep_runs` records, and
135    /// delete every entry none of them reference. Returns the number of entries deleted.
136    pub fn record_run_and_prune(
137        &self,
138        used: impl IntoIterator<Item = String>,
139        keep_runs: usize,
140    ) -> Result<usize> {
141        let runs_path = self.dir.join("runs.json");
142        let mut runs: RunsFile = std::fs::read(&runs_path)
143            .ok()
144            .and_then(|b| serde_json::from_slice(&b).ok())
145            .unwrap_or_default();
146        let mut keys: Vec<String> = used.into_iter().collect();
147        keys.sort();
148        keys.dedup();
149        runs.runs.push(RunRecord { keys });
150        let keep = keep_runs.max(1);
151        if runs.runs.len() > keep {
152            let drop = runs.runs.len() - keep;
153            runs.runs.drain(..drop);
154        }
155
156        std::fs::create_dir_all(&self.dir)?;
157        let tmp = tmp_path_for(&runs_path);
158        std::fs::write(&tmp, serde_json::to_vec_pretty(&runs)?)?;
159        atomic_replace(&tmp, &runs_path)?;
160
161        let live: BTreeSet<&str> = runs
162            .runs
163            .iter()
164            .flat_map(|r| r.keys.iter().map(String::as_str))
165            .collect();
166        let mut deleted = 0;
167        let root = self.dir.join(LAYOUT);
168        let Ok(shards) = std::fs::read_dir(&root) else {
169            return Ok(0);
170        };
171        for shard in shards.flatten() {
172            let Ok(files) = std::fs::read_dir(shard.path()) else {
173                continue;
174            };
175            for file in files.flatten() {
176                let path = file.path();
177                let Some(stem) = path.file_stem().and_then(|s| s.to_str()) else {
178                    continue;
179                };
180                let is_entry = path.extension().is_some_and(|e| e == "json");
181                if is_entry && !live.contains(stem) && std::fs::remove_file(&path).is_ok() {
182                    deleted += 1;
183                }
184            }
185        }
186        Ok(deleted)
187    }
188
189    /// Number of entries on disk.
190    pub fn count(&self) -> usize {
191        let Ok(shards) = std::fs::read_dir(self.dir.join(LAYOUT)) else {
192            return 0;
193        };
194        shards
195            .flatten()
196            .filter_map(|s| std::fs::read_dir(s.path()).ok())
197            .map(|files| {
198                files
199                    .flatten()
200                    .filter(|f| f.path().extension().is_some_and(|e| e == "json"))
201                    .count()
202            })
203            .sum()
204    }
205}
206
207#[cfg(test)]
208mod tests {
209    use super::*;
210    use tempfile::TempDir;
211
212    fn material<'a>(user: &'a str, model: Option<&'a str>) -> KeyMaterial<'a> {
213        KeyMaterial {
214            task_kind: "modules",
215            prompt_version: 1,
216            provider: model.map(|_| "anthropic"),
217            model,
218            max_tokens: 600,
219            output_mode: "text",
220            output_schema: None,
221            system: "system prompt",
222            user,
223        }
224    }
225
226    fn entry(key: &str) -> CacheEntry {
227        CacheEntry {
228            key: key.to_string(),
229            task_id: "module:src".into(),
230            task_kind: "modules".into(),
231            model: "m".into(),
232            usage: None,
233            text: "hello".into(),
234        }
235    }
236
237    #[test]
238    fn key_is_pinned() {
239        // A change here invalidates every user's cache. Bump KEY_SCHEMA on purpose if so.
240        assert_eq!(
241            material("ctx", Some("claude-sonnet-5")).key(),
242            "bf44ce3c02d91bff03615e401af9e65d5d33cfc9229d206b7f7f341466aa2815"
243        );
244    }
245
246    #[test]
247    fn key_depends_on_request_not_time() {
248        let a = material("ctx", Some("m1")).key();
249        assert_eq!(a, material("ctx", Some("m1")).key());
250        assert_ne!(a, material("ctx2", Some("m1")).key());
251        assert_ne!(a, material("ctx", Some("m2")).key());
252        assert_ne!(a, material("ctx", None).key());
253        let mut v2 = material("ctx", Some("m1"));
254        v2.prompt_version = 2;
255        assert_ne!(a, v2.key());
256    }
257
258    #[test]
259    fn put_get_round_trip_and_miss() {
260        let dir = TempDir::new().unwrap();
261        let cache = WriteCache::new(dir.path());
262        let key = material("ctx", Some("m")).key();
263        assert!(cache.get(&key).is_none());
264        cache.put(&entry(&key)).unwrap();
265        assert_eq!(cache.get(&key), Some(entry(&key)));
266        assert_eq!(cache.count(), 1);
267        assert!(
268            cache
269                .entry_path(&key)
270                .starts_with(dir.path().join("v1").join(&key[..2]))
271        );
272    }
273
274    #[test]
275    fn corrupt_entry_is_a_miss() {
276        let dir = TempDir::new().unwrap();
277        let cache = WriteCache::new(dir.path());
278        let key = material("ctx", Some("m")).key();
279        let path = cache.entry_path(&key);
280        std::fs::create_dir_all(path.parent().unwrap()).unwrap();
281        std::fs::write(&path, b"{not json").unwrap();
282        assert!(cache.get(&key).is_none());
283    }
284
285    #[test]
286    fn prune_keeps_last_k_runs() {
287        let dir = TempDir::new().unwrap();
288        let cache = WriteCache::new(dir.path());
289        let keys: Vec<String> = (0..4)
290            .map(|i| material(&format!("ctx{i}"), Some("m")).key())
291            .collect();
292        for k in &keys {
293            cache.put(&entry(k)).unwrap();
294        }
295        // keep_runs = 2. Run A uses {0,1}; run B uses {2}; run C uses {3}.
296        // After C, the kept runs are B and C, so 0 and 1 go.
297        let (k0, k1, k2, k3) = (&keys[0], &keys[1], &keys[2], &keys[3]);
298        assert_eq!(
299            cache
300                .record_run_and_prune([k0.clone(), k1.clone()], 2)
301                .unwrap(),
302            2,
303            "run A: 2 and 3 are unreferenced"
304        );
305        cache.put(&entry(k2)).unwrap();
306        cache.put(&entry(k3)).unwrap();
307        assert_eq!(
308            cache.record_run_and_prune([k2.clone()], 2).unwrap(),
309            1,
310            "run B: 3"
311        );
312        cache.put(&entry(k3)).unwrap();
313        assert_eq!(
314            cache.record_run_and_prune([k3.clone()], 2).unwrap(),
315            2,
316            "run C: 0, 1"
317        );
318        assert!(cache.get(k0).is_none());
319        assert!(cache.get(k1).is_none());
320        assert!(cache.get(k2).is_some());
321        assert!(cache.get(k3).is_some());
322    }
323}