Skip to main content

second_brain_core/
model_cache.rs

1use std::ffi::OsString;
2use std::path::{Path, PathBuf};
3
4use anyhow::{Context, Result, bail};
5use fastembed::{RerankerModel, TextRerank};
6
7use crate::retrieval::rerank::parse_model;
8
9pub const MODEL_CACHE_DIR_ENV: &str = "FASTEMBED_CACHE_DIR";
10
11/// Files fastembed's tokenizer loader reads from a model snapshot, beside the
12/// model's own ONNX files.
13pub const TOKENIZER_FILES: [&str; 4] = [
14    "tokenizer.json",
15    "config.json",
16    "tokenizer_config.json",
17    "special_tokens_map.json",
18];
19
20/// The reranker the daemon loads when `SECOND_BRAIN_RERANK_MODEL` is unset.
21pub const DEFAULT_RERANK_MODEL: &str = "jina-reranker-v1-turbo";
22
23/// A Hugging Face model repo and the files, relative to a snapshot, that must
24/// all exist before the model can load.
25#[derive(Debug, Clone, PartialEq, Eq)]
26pub struct CachedModel {
27    pub code: String,
28    pub files: Vec<String>,
29    lookup: SnapshotLookup,
30}
31
32#[derive(Debug, Clone, Copy, PartialEq, Eq)]
33pub(crate) enum SnapshotLookup {
34    /// Loaded through hf-hub, which finds cached files only through
35    /// `<repo>/refs/main`.
36    MainRef,
37    /// Loaded straight from any complete snapshot directory.
38    AnyComplete,
39}
40
41impl CachedModel {
42    pub(crate) fn new(
43        code: &str,
44        model_file: &str,
45        additional_files: &[String],
46        lookup: SnapshotLookup,
47    ) -> Self {
48        let files = std::iter::once(model_file.to_string())
49            .chain(additional_files.iter().cloned())
50            .chain(TOKENIZER_FILES.iter().map(|f| f.to_string()))
51            .collect();
52        Self {
53            code: code.to_string(),
54            files,
55            lookup,
56        }
57    }
58
59    /// The snapshot under `cache_dir` the daemon loads this model from without
60    /// downloading; `None` when loading it would download again.
61    pub fn loadable_snapshot(&self, cache_dir: &Path) -> Option<PathBuf> {
62        match self.lookup {
63            SnapshotLookup::MainRef => main_ref_snapshot(cache_dir, self),
64            SnapshotLookup::AnyComplete => cached_snapshot(cache_dir, self),
65        }
66    }
67}
68
69pub(crate) fn reranker_model(model: &RerankerModel) -> CachedModel {
70    let info = TextRerank::get_model_info(model);
71    CachedModel::new(
72        &info.model_code,
73        &info.model_file,
74        &info.additional_files,
75        SnapshotLookup::AnyComplete,
76    )
77}
78
79pub fn default_reranker_model() -> Result<CachedModel> {
80    Ok(reranker_model(&parse_model(DEFAULT_RERANK_MODEL)?))
81}
82
83/// `<cache_dir>/models--<org>--<repo>/snapshots`, the Hugging Face hub cache
84/// layout fastembed and hf-hub download into.
85pub fn snapshots_dir(cache_dir: &Path, model_code: &str) -> PathBuf {
86    cache_dir
87        .join(format!("models--{}", model_code.replace('/', "--")))
88        .join("snapshots")
89}
90
91/// The first snapshot, in name order, of `model` under `cache_dir` that holds
92/// every one of its files; `None` when no download finished.
93pub fn cached_snapshot(cache_dir: &Path, model: &CachedModel) -> Option<PathBuf> {
94    let mut snapshots: Vec<PathBuf> = std::fs::read_dir(snapshots_dir(cache_dir, &model.code))
95        .ok()?
96        .filter_map(|entry| entry.ok())
97        .map(|entry| entry.path())
98        .filter(|path| path.is_dir())
99        .collect();
100    snapshots.sort();
101    snapshots
102        .into_iter()
103        .find(|snapshot| holds_every_file(snapshot, model))
104}
105
106/// The snapshot `<repo>/refs/main` names, when it holds every file of `model`.
107/// hf-hub, and so fastembed, finds cached files only through that ref, read
108/// untrimmed, and downloads again without it.
109pub fn main_ref_snapshot(cache_dir: &Path, model: &CachedModel) -> Option<PathBuf> {
110    let snapshots = snapshots_dir(cache_dir, &model.code);
111    let main_ref = snapshots.parent()?.join("refs").join("main");
112    let revision = std::fs::read_to_string(main_ref).ok()?;
113    let snapshot = snapshots.join(revision);
114    holds_every_file(&snapshot, model).then_some(snapshot)
115}
116
117fn holds_every_file(snapshot: &Path, model: &CachedModel) -> bool {
118    model.files.iter().all(|file| snapshot.join(file).is_file())
119}
120
121/// Directory the embedding and reranker models are downloaded to and loaded
122/// from: `FASTEMBED_CACHE_DIR` when set, otherwise `second-brain/models` under
123/// the OS user cache dir. Created if missing. Errors instead of resolving to a
124/// path relative to the working directory.
125pub fn model_cache_dir() -> Result<PathBuf> {
126    let dir = resolve(std::env::var_os(MODEL_CACHE_DIR_ENV), dirs::cache_dir())?;
127    std::fs::create_dir_all(&dir)
128        .with_context(|| format!("creating model cache dir {}", dir.display()))?;
129    Ok(dir)
130}
131
132/// The model cache inside a user cache dir, where a daemon whose environment
133/// does not set `FASTEMBED_CACHE_DIR` keeps its models.
134pub fn model_cache_dir_in(user_cache: &Path) -> PathBuf {
135    user_cache.join("second-brain").join("models")
136}
137
138fn resolve(env_override: Option<OsString>, user_cache: Option<PathBuf>) -> Result<PathBuf> {
139    if let Some(value) = env_override.filter(|v| !v.is_empty()) {
140        let dir = PathBuf::from(value);
141        if !dir.is_absolute() {
142            bail!(
143                "{MODEL_CACHE_DIR_ENV} must be an absolute path, got {}",
144                dir.display()
145            );
146        }
147        return Ok(dir);
148    }
149    let user_cache = user_cache.with_context(|| {
150        format!("cannot determine the user cache directory; set {MODEL_CACHE_DIR_ENV}")
151    })?;
152    if !user_cache.is_absolute() {
153        bail!(
154            "user cache directory {} is not absolute; set {MODEL_CACHE_DIR_ENV}",
155            user_cache.display()
156        );
157    }
158    Ok(model_cache_dir_in(&user_cache))
159}
160
161#[cfg(test)]
162pub(crate) mod test_support {
163    use std::ffi::OsString;
164    use std::path::Path;
165    use std::sync::{Mutex, MutexGuard};
166
167    use super::MODEL_CACHE_DIR_ENV;
168
169    static ENV_LOCK: Mutex<()> = Mutex::new(());
170
171    /// Points `FASTEMBED_CACHE_DIR` at `dir` while held and restores the
172    /// previous value on drop. Holds a crate-wide lock, because the variable
173    /// is process-global and unit tests run on parallel threads.
174    pub(crate) struct CacheDirOverride {
175        previous: Option<OsString>,
176        _lock: MutexGuard<'static, ()>,
177    }
178
179    impl CacheDirOverride {
180        pub(crate) fn new(dir: &Path) -> Self {
181            let lock = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
182            let previous = std::env::var_os(MODEL_CACHE_DIR_ENV);
183            // SAFETY: every test that touches this variable holds ENV_LOCK.
184            unsafe { std::env::set_var(MODEL_CACHE_DIR_ENV, dir) };
185            Self {
186                previous,
187                _lock: lock,
188            }
189        }
190    }
191
192    impl Drop for CacheDirOverride {
193        fn drop(&mut self) {
194            // SAFETY: ENV_LOCK is still held until this guard's fields drop.
195            unsafe {
196                match &self.previous {
197                    Some(value) => std::env::set_var(MODEL_CACHE_DIR_ENV, value),
198                    None => std::env::remove_var(MODEL_CACHE_DIR_ENV),
199                }
200            }
201        }
202    }
203}
204
205#[cfg(test)]
206mod tests {
207    use super::test_support::CacheDirOverride;
208    use super::*;
209
210    fn jina() -> CachedModel {
211        reranker_model(&RerankerModel::JINARerankerV1TurboEn)
212    }
213
214    fn seed(snapshot: &Path, files: &[String]) {
215        for file in files {
216            let path = snapshot.join(file);
217            std::fs::create_dir_all(path.parent().unwrap()).unwrap();
218            std::fs::write(path, b"x").unwrap();
219        }
220    }
221
222    fn hub_snapshot(cache: &Path, repo_dir: &str, revision: &str) -> PathBuf {
223        cache.join(repo_dir).join("snapshots").join(revision)
224    }
225
226    #[test]
227    fn cached_snapshot_finds_a_snapshot_holding_every_required_file() {
228        let tmp = tempfile::tempdir().unwrap();
229        let model = jina();
230        let snap = hub_snapshot(
231            tmp.path(),
232            "models--jinaai--jina-reranker-v1-turbo-en",
233            "rev1",
234        );
235        seed(&snap, &model.files);
236
237        assert_eq!(cached_snapshot(tmp.path(), &model), Some(snap));
238    }
239
240    #[test]
241    fn cached_snapshot_is_none_when_tokenizer_json_is_missing() {
242        let tmp = tempfile::tempdir().unwrap();
243        let model = jina();
244        let snap = hub_snapshot(
245            tmp.path(),
246            "models--jinaai--jina-reranker-v1-turbo-en",
247            "rev1",
248        );
249        seed(&snap, &model.files);
250        assert_eq!(cached_snapshot(tmp.path(), &model), Some(snap.clone()));
251        std::fs::remove_file(snap.join("tokenizer.json")).unwrap();
252
253        assert_eq!(cached_snapshot(tmp.path(), &model), None);
254    }
255
256    #[test]
257    fn cached_snapshot_is_none_when_the_onnx_file_is_missing() {
258        let tmp = tempfile::tempdir().unwrap();
259        let model = jina();
260        let snap = hub_snapshot(
261            tmp.path(),
262            "models--jinaai--jina-reranker-v1-turbo-en",
263            "rev1",
264        );
265        seed(&snap, &model.files);
266        assert_eq!(cached_snapshot(tmp.path(), &model), Some(snap.clone()));
267        std::fs::remove_file(snap.join("onnx").join("model.onnx")).unwrap();
268
269        assert_eq!(cached_snapshot(tmp.path(), &model), None);
270    }
271
272    #[test]
273    fn cached_snapshot_skips_an_interrupted_snapshot_for_a_complete_one() {
274        let tmp = tempfile::tempdir().unwrap();
275        let model = jina();
276        let repo = "models--jinaai--jina-reranker-v1-turbo-en";
277        seed(
278            &hub_snapshot(tmp.path(), repo, "a-partial"),
279            &model.files[..1],
280        );
281        let complete = hub_snapshot(tmp.path(), repo, "b-complete");
282        seed(&complete, &model.files);
283
284        assert_eq!(cached_snapshot(tmp.path(), &model), Some(complete));
285    }
286
287    #[test]
288    fn cached_snapshot_is_none_when_nothing_was_downloaded() {
289        let tmp = tempfile::tempdir().unwrap();
290
291        assert_eq!(cached_snapshot(tmp.path(), &jina()), None);
292        assert_eq!(cached_snapshot(&tmp.path().join("absent"), &jina()), None);
293    }
294
295    fn embedder() -> CachedModel {
296        crate::embedding::cached_model().unwrap()
297    }
298
299    fn repo_snapshot(cache: &Path, model: &CachedModel, revision: &str) -> PathBuf {
300        snapshots_dir(cache, &model.code).join(revision)
301    }
302
303    fn point_main_at(cache: &Path, model: &CachedModel, contents: &str) {
304        let refs = snapshots_dir(cache, &model.code)
305            .parent()
306            .unwrap()
307            .join("refs");
308        std::fs::create_dir_all(&refs).unwrap();
309        std::fs::write(refs.join("main"), contents).unwrap();
310    }
311
312    #[test]
313    fn main_ref_snapshot_is_the_complete_snapshot_refs_main_names() {
314        let tmp = tempfile::tempdir().unwrap();
315        let model = embedder();
316        seed(&repo_snapshot(tmp.path(), &model, "a-older"), &model.files);
317        let main = repo_snapshot(tmp.path(), &model, "b-main");
318        seed(&main, &model.files);
319        point_main_at(tmp.path(), &model, "b-main");
320
321        assert_eq!(main_ref_snapshot(tmp.path(), &model), Some(main));
322    }
323
324    #[test]
325    fn main_ref_snapshot_is_none_without_refs_main() {
326        let tmp = tempfile::tempdir().unwrap();
327        let model = embedder();
328        seed(&repo_snapshot(tmp.path(), &model, "rev1"), &model.files);
329
330        assert_eq!(main_ref_snapshot(tmp.path(), &model), None);
331    }
332
333    #[test]
334    fn main_ref_snapshot_is_none_when_refs_main_names_an_incomplete_snapshot() {
335        let tmp = tempfile::tempdir().unwrap();
336        let model = embedder();
337        seed(
338            &repo_snapshot(tmp.path(), &model, "a-complete"),
339            &model.files,
340        );
341        let main = repo_snapshot(tmp.path(), &model, "b-main");
342        seed(&main, &model.files);
343        std::fs::remove_file(main.join("tokenizer.json")).unwrap();
344        point_main_at(tmp.path(), &model, "b-main");
345
346        assert_eq!(main_ref_snapshot(tmp.path(), &model), None);
347    }
348
349    #[test]
350    fn main_ref_snapshot_is_none_when_refs_main_names_a_missing_snapshot() {
351        let tmp = tempfile::tempdir().unwrap();
352        let model = embedder();
353        seed(
354            &repo_snapshot(tmp.path(), &model, "a-complete"),
355            &model.files,
356        );
357        point_main_at(tmp.path(), &model, "b-gone");
358
359        assert_eq!(main_ref_snapshot(tmp.path(), &model), None);
360    }
361
362    #[test]
363    fn main_ref_snapshot_reads_refs_main_verbatim_like_hf_hub() {
364        let tmp = tempfile::tempdir().unwrap();
365        let model = embedder();
366        seed(&repo_snapshot(tmp.path(), &model, "rev1"), &model.files);
367        point_main_at(tmp.path(), &model, "rev1\n");
368
369        assert_eq!(main_ref_snapshot(tmp.path(), &model), None);
370    }
371
372    #[test]
373    fn the_embedders_loadable_snapshot_needs_refs_main() {
374        let tmp = tempfile::tempdir().unwrap();
375        let model = embedder();
376        let snap = repo_snapshot(tmp.path(), &model, "rev1");
377        seed(&snap, &model.files);
378        assert_eq!(model.loadable_snapshot(tmp.path()), None);
379
380        point_main_at(tmp.path(), &model, "rev1");
381
382        assert_eq!(model.loadable_snapshot(tmp.path()), Some(snap));
383    }
384
385    #[test]
386    fn the_rerankers_loadable_snapshot_is_any_complete_snapshot() {
387        let tmp = tempfile::tempdir().unwrap();
388        let model = jina();
389        let snap = repo_snapshot(tmp.path(), &model, "rev1");
390        seed(&snap, &model.files);
391
392        assert_eq!(model.loadable_snapshot(tmp.path()), Some(snap));
393    }
394
395    #[test]
396    fn snapshots_dir_follows_the_hugging_face_hub_layout() {
397        assert_eq!(
398            snapshots_dir(Path::new("/cache"), "jinaai/jina-reranker-v1-turbo-en"),
399            PathBuf::from("/cache/models--jinaai--jina-reranker-v1-turbo-en/snapshots")
400        );
401    }
402
403    #[test]
404    fn default_reranker_model_is_fastembeds_entry_for_the_daemon_default() {
405        let info = TextRerank::get_model_info(&parse_model(DEFAULT_RERANK_MODEL).unwrap());
406
407        let model = default_reranker_model().unwrap();
408
409        assert_eq!(model.code, info.model_code);
410        assert!(model.files.contains(&info.model_file), "{model:?}");
411        for file in &info.additional_files {
412            assert!(model.files.contains(file), "{model:?}");
413        }
414        for file in TOKENIZER_FILES {
415            assert!(model.files.iter().any(|f| f == file), "{model:?}");
416        }
417    }
418
419    #[test]
420    fn unset_override_resolves_under_the_os_user_cache_dir() {
421        let user_cache = dirs::cache_dir().expect("test host has a user cache dir");
422
423        let dir = resolve(None, Some(user_cache.clone())).unwrap();
424
425        assert_eq!(dir, user_cache.join("second-brain").join("models"));
426        assert!(dir.is_absolute());
427    }
428
429    #[test]
430    fn set_override_wins_over_the_user_cache_dir() {
431        let dir = resolve(
432            Some(OsString::from("/srv/models")),
433            Some(PathBuf::from("/home/u/.cache")),
434        )
435        .unwrap();
436
437        assert_eq!(dir, PathBuf::from("/srv/models"));
438    }
439
440    #[test]
441    fn empty_override_counts_as_unset() {
442        let dir = resolve(Some(OsString::new()), Some(PathBuf::from("/home/u/.cache"))).unwrap();
443
444        assert_eq!(dir, PathBuf::from("/home/u/.cache/second-brain/models"));
445    }
446
447    #[test]
448    fn relative_override_is_rejected() {
449        let err = resolve(
450            Some(OsString::from(".fastembed_cache")),
451            Some(PathBuf::from("/home/u/.cache")),
452        )
453        .unwrap_err();
454
455        assert!(err.to_string().contains(MODEL_CACHE_DIR_ENV));
456    }
457
458    #[test]
459    fn missing_user_cache_dir_is_an_error_not_a_cwd_fallback() {
460        let err = resolve(None, None).unwrap_err();
461
462        assert!(err.to_string().contains(MODEL_CACHE_DIR_ENV));
463    }
464
465    #[test]
466    fn relative_user_cache_dir_is_rejected() {
467        let err = resolve(None, Some(PathBuf::from("home/.cache"))).unwrap_err();
468
469        assert!(err.to_string().contains(MODEL_CACHE_DIR_ENV));
470    }
471
472    #[test]
473    fn model_cache_dir_returns_and_creates_the_override_dir() {
474        let tmp = tempfile::tempdir().unwrap();
475        let target = tmp.path().join("nested").join("models");
476        let _env = CacheDirOverride::new(&target);
477
478        let dir = model_cache_dir().unwrap();
479
480        assert_eq!(dir, target);
481        assert!(dir.is_dir());
482    }
483}