Skip to main content

second_brain_api/
models.rs

1//! Startup of the search indexes and the embedding and rerank models, and the
2//! timers that unload idle models.
3
4use std::path::Path;
5use std::sync::{Arc, RwLock};
6use std::time::Duration;
7
8use second_brain_core::embedding::{Activations, Embedder};
9use second_brain_core::index::{IndexClaim, Indexes};
10use second_brain_core::model_cache::DEFAULT_RERANK_MODEL;
11use second_brain_core::residency;
12use second_brain_core::retrieval::Reranker;
13use second_brain_core::sqlite_store::SqliteStore;
14
15/// The reader is what `AppState` and every recall call see; the writer is
16/// private to the drain loop. Splitting them means recall never contends
17/// with the drain loop for anything heavier than the brief `reload_view`
18/// call the loop makes on the reader after each successful drain.
19pub struct IndexHandles {
20    pub reader: Arc<RwLock<Indexes>>,
21    pub writer: Indexes,
22    pub claim: IndexClaim,
23}
24
25/// `on_rebuild` hears how many memories a rebuild covers before it starts.
26pub fn open_index_handles(
27    dir: &Path,
28    store: &SqliteStore,
29    on_rebuild: impl FnOnce(usize),
30) -> anyhow::Result<IndexHandles> {
31    let claim = IndexClaim::acquire(dir)?;
32    let mut writer = Indexes::open(dir)?;
33    if writer.consistent_with(store)? {
34        tracing::info!("search indexes consistent at {}", dir.display());
35        // `rebuild` persists as its last step, but the "already consistent"
36        // branch never mutates, so a brand-new index (nothing on disk yet,
37        // trivially consistent with an empty store) would otherwise leave
38        // `open_readonly` below with no files to open.
39        writer.persist()?;
40    } else {
41        tracing::info!("rebuilding search indexes at {}", dir.display());
42        match store.indexable_count() {
43            Ok(n) => on_rebuild(n),
44            Err(e) => tracing::warn!("cannot count the memories to rebuild: {e:#}"),
45        }
46        let n = writer.rebuild(store)?;
47        tracing::info!("search indexes rebuilt: {n} memories");
48        residency::release_free_heap();
49    }
50    let reader = Indexes::open_readonly(dir)?;
51    Ok(IndexHandles {
52        reader: Arc::new(RwLock::new(reader)),
53        writer,
54        claim,
55    })
56}
57
58/// A rerank pool of 1 (the old default) serializes every concurrent recall
59/// behind one session; `evals/rerank-cost-curve.md` reproduced the result
60/// of that under ordinary eval load: a queued request timed out at 30s.
61/// `Reranker`'s intra-op thread count is already sized per-session as
62/// `cores / pool_size` (see `crates/core/src/retrieval/rerank.rs`), so a
63/// larger pool does not add CPU throughput the machine doesn't have, and
64/// wave 1's measurement found oversubscribed sessions just contend for the
65/// same cores. What a larger pool buys is concurrency headroom: enough
66/// sessions that an ordinary burst of overlapping recalls queues briefly
67/// instead of erroring. Capping at 4 keeps that headroom without loading
68/// more model copies into memory than a handful of concurrent clients would
69/// ever need.
70fn rerank_pool_size() -> usize {
71    match std::env::var("SECOND_BRAIN_RERANK_POOL") {
72        Ok(v) => v
73            .parse::<usize>()
74            .unwrap_or_else(|_| {
75                panic!("invalid SECOND_BRAIN_RERANK_POOL '{v}': expected a positive integer")
76            })
77            .max(1),
78        Err(_) => {
79            let cores = std::thread::available_parallelism()
80                .map(|n| n.get())
81                .unwrap_or(1);
82            cores.clamp(1, 4)
83        }
84    }
85}
86
87pub fn load_embedder() -> Option<Arc<Embedder>> {
88    match Embedder::with_sessions_and_activations(1, Activations::Released) {
89        Ok(e) => {
90            tracing::info!("embedding model loaded");
91            Some(Arc::new(e))
92        }
93        Err(e) => {
94            tracing::warn!("embedding model unavailable: {e}");
95            None
96        }
97    }
98}
99
100pub fn load_reranker(indexes_open: bool) -> Option<Arc<Reranker>> {
101    let rerank_model = std::env::var("SECOND_BRAIN_RERANK_MODEL")
102        .unwrap_or_else(|_| DEFAULT_RERANK_MODEL.to_string());
103    if !indexes_open || rerank_model == "off" {
104        None
105    } else {
106        let pool_size = rerank_pool_size();
107        match Reranker::with_sessions(&rerank_model, pool_size) {
108            Ok(r) => {
109                tracing::info!("rerank model {rerank_model} loaded ({pool_size} sessions)");
110                Some(Arc::new(r))
111            }
112            Err(e) => {
113                tracing::warn!("rerank model {rerank_model} unavailable: {e:#}");
114                None
115            }
116        }
117    }
118}
119
120/// Must run inside the tokio runtime.
121pub fn spawn_idle_unload(embedder: Option<Arc<Embedder>>, reranker: Option<Arc<Reranker>>) {
122    if let Some(emb) = embedder {
123        tokio::spawn(async move {
124            let mut interval = tokio::time::interval(Duration::from_secs(60));
125            loop {
126                interval.tick().await;
127                emb.drop_if_idle(300);
128            }
129        });
130    }
131
132    if let Some(r) = reranker {
133        tokio::spawn(async move {
134            let mut interval = tokio::time::interval(Duration::from_secs(60));
135            loop {
136                interval.tick().await;
137                r.drop_if_idle(300);
138            }
139        });
140    }
141}
142
143#[cfg(test)]
144mod tests {
145    use super::*;
146    use second_brain_core::schema::{Memory, MemoryType};
147    use second_brain_core::store::Store;
148
149    #[test]
150    fn a_held_claim_refuses_a_second_daemon_until_dropped() {
151        let dir = tempfile::tempdir().unwrap();
152        let store = SqliteStore::in_memory("m".to_string()).unwrap();
153        let first = open_index_handles(dir.path(), &store, |_| {}).unwrap();
154        let err = open_index_handles(dir.path(), &store, |_| {})
155            .err()
156            .unwrap();
157        assert!(
158            format!("{err:#}").contains("held by another daemon"),
159            "{err:#}"
160        );
161        drop(first);
162        open_index_handles(dir.path(), &store, |_| {}).unwrap();
163    }
164
165    #[test]
166    fn a_rebuild_is_announced_with_its_size_and_a_consistent_open_is_not() {
167        let dir = tempfile::tempdir().unwrap();
168        let store = SqliteStore::in_memory("m".to_string()).unwrap();
169        for content in ["first note", "second note"] {
170            let m = Memory::new(
171                content.to_string(),
172                MemoryType::Semantic,
173                "test".to_string(),
174                String::new(),
175            );
176            store.store_memory(&m).unwrap();
177        }
178
179        let mut announced = None;
180        drop(open_index_handles(dir.path(), &store, |n| announced = Some(n)).unwrap());
181        assert_eq!(announced, Some(2));
182
183        let mut again = None;
184        drop(open_index_handles(dir.path(), &store, |n| again = Some(n)).unwrap());
185        assert_eq!(again, None);
186    }
187}