reflex-search 2.2.0

A local-first, structure-aware code search engine for AI agents
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
//! A shared, reusable handle on the open index files.
//!
//! Before this module, every query re-opened `content.bin` and `trigrams.bin`
//! (three to four times per query, once per phase), parsed the whole trigram
//! directory on each open, and ran `PRAGMA quick_check` over `meta.db`. On a
//! 29 MB corpus that put a ~50 ms floor under every call, including a zero-hit
//! search through the resident `rfx mcp` server.
//!
//! [`OpenIndex`] holds the [`IndexSnapshot`] (the memory maps of every store the
//! manifest names), a path→file-id map, and a rayon pool sized from
//! `[performance] parallel_threads`. Handles live in a process-wide registry keyed
//! by the canonical cache directory, so the MCP server, the HTTP server, and the
//! CLI all share one open per index. A handle is reused while `manifest.json` and
//! the `content.bin` / `trigrams.bin` links carry the same fingerprint (device,
//! inode, size, mtime); an indexer run invalidates it explicitly, and an
//! `rfx index` from another process is caught by the fingerprint compare on the
//! next lookup (every publish renames a new manifest into place).

use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex, OnceLock};

use anyhow::{Context, Result};

use crate::cache::CacheManager;
use crate::errors::ReflexError;
use crate::snapshot::IndexSnapshot;

/// Identity of one on-disk file, cheap to read (one `stat`).
#[derive(Debug, Clone, PartialEq, Eq)]
struct FileStamp {
    dev: u64,
    ino: u64,
    size: u64,
    mtime: Option<std::time::SystemTime>,
}

impl FileStamp {
    fn of(path: &Path) -> std::io::Result<Self> {
        let md = std::fs::metadata(path)?;
        #[cfg(unix)]
        let (dev, ino) = {
            use std::os::unix::fs::MetadataExt;
            (md.dev(), md.ino())
        };
        #[cfg(not(unix))]
        let (dev, ino) = (0u64, 0u64);
        Ok(Self {
            dev,
            ino,
            size: md.len(),
            mtime: md.modified().ok(),
        })
    }
}

/// Fingerprint of the snapshot a handle was opened from: the manifest (renamed
/// into place by every publish) and the `content.bin` / `trigrams.bin` links (a
/// cache without a manifest has only those). Any of them may be absent.
#[derive(Debug, Clone, PartialEq, Eq)]
struct Fingerprint {
    manifest: Option<FileStamp>,
    content: Option<FileStamp>,
    trigrams: Option<FileStamp>,
}

impl Fingerprint {
    fn current(cache_dir: &Path) -> std::io::Result<Self> {
        let manifest = FileStamp::of(&cache_dir.join(crate::snapshot::MANIFEST)).ok();
        let content = FileStamp::of(&cache_dir.join(crate::snapshot::FIXED_CONTENT)).ok();
        let trigrams = FileStamp::of(&cache_dir.join(crate::snapshot::FIXED_TRIGRAMS)).ok();
        if manifest.is_none() && content.is_none() {
            return Err(std::io::Error::new(
                std::io::ErrorKind::NotFound,
                "no manifest.json and no content.bin",
            ));
        }
        Ok(Self {
            manifest,
            content,
            trigrams,
        })
    }
}

/// Everything a query needs from the index, opened once.
pub struct OpenIndex {
    cache_dir: PathBuf,
    /// The stores: content, paths and candidate lookups by file id.
    pub snapshot: IndexSnapshot,
    /// `path → file_id` of live files, with any leading `./` stripped, built on first use.
    /// Only symbol and AST queries need it; a full-text query never pays for it.
    path_to_id: OnceLock<HashMap<String, u32>>,
    /// Query-side thread pool, sized from `[performance] parallel_threads`, built
    /// on first parallel use: a zero-hit query (no candidates to verify) never
    /// spawns a thread, and neither does a `check_index_status` call.
    pool: OnceLock<rayon::ThreadPool>,
    threads: usize,
    /// `IndexConfig::max_posting_list_entries` at open time (0 = unlimited).
    posting_cap: usize,
    fingerprint: Fingerprint,
    /// One `meta.db` connection for the query path, opened on first use with the
    /// symbol-cache schema ensured. Symbol queries opened three to four
    /// connections per call (each running the WAL and foreign-key pragmas) and
    /// re-ran the schema migration every time.
    meta: OnceLock<Mutex<rusqlite::Connection>>,
    /// Every indexed file's fingerprint, loaded on first use by the freshness walk
    /// (outside git). Dropped with the handle when the index is rewritten.
    fingerprints: OnceLock<Arc<HashMap<String, crate::cache::FileFingerprint>>>,
}

impl std::fmt::Debug for OpenIndex {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("OpenIndex")
            .field("cache_dir", &self.cache_dir)
            .field("files", &self.snapshot.live_file_count())
            .field("threads", &self.threads)
            .finish()
    }
}

/// Upper bound on the automatic thread count for query-time verification.
///
/// The indexer caps itself at 8 to limit cache contention while it writes; a
/// read-only verification pass scales further, and ripgrep's 5x from 16 threads
/// on the field-test box is the number to match.
const QUERY_AUTO_THREAD_CAP: usize = 32;

impl OpenIndex {
    fn open(cache: &CacheManager, fingerprint: Fingerprint) -> Result<Self> {
        let cache_dir = cache.path().to_path_buf();
        let snapshot = IndexSnapshot::open(&cache_dir)?;

        let config = cache.load_index_config().unwrap_or_else(|e| {
            log::debug!("Using default index config for query pool: {}", e);
            crate::models::IndexConfig::default()
        });
        let threads =
            crate::models::resolve_thread_count(config.parallel_threads, QUERY_AUTO_THREAD_CAP);

        log::debug!(
            "Opened index {}: {} files, {} trigrams, {} query threads",
            cache_dir.display(),
            snapshot.live_file_count(),
            snapshot.trigram_count(),
            threads
        );

        Ok(Self {
            cache_dir,
            snapshot,
            path_to_id: OnceLock::new(),
            pool: OnceLock::new(),
            threads,
            posting_cap: config.max_posting_list_entries,
            fingerprint,
            meta: OnceLock::new(),
            fingerprints: OnceLock::new(),
        })
    }

    /// Whether meta.db's rows go with this handle's snapshot (same generation).
    /// Between an index run's two commits (manifest, then meta.db) they do not.
    /// A cache without a manifest has nothing to compare and always matches.
    pub fn meta_matches_snapshot(&self, conn: &rusqlite::Connection) -> bool {
        let Some(generation) = self.snapshot.generation() else {
            return true;
        };
        crate::meta_update::get_statistic(conn, crate::indexer::INDEX_GENERATION_KEY)
            .ok()
            .flatten()
            .and_then(|g| g.parse::<u64>().ok())
            == Some(generation)
    }

    /// The fingerprint table, read once per handle (not memoised while meta.db is
    /// behind the snapshot, so the next call sees the committed rows).
    pub fn fingerprints(
        &self,
        cache: &CacheManager,
    ) -> Result<Arc<HashMap<String, crate::cache::FileFingerprint>>> {
        if let Some(fp) = self.fingerprints.get() {
            return Ok(Arc::clone(fp));
        }
        let loaded = Arc::new(cache.load_fingerprints()?);
        let in_step = self
            .meta_conn()
            .map(|conn| self.meta_matches_snapshot(&conn))
            .unwrap_or(false);
        if !in_step {
            return Ok(loaded);
        }
        // A concurrent first caller may have won the race; either table is fine.
        let _ = self.fingerprints.set(Arc::clone(&loaded));
        Ok(Arc::clone(self.fingerprints.get().expect("set above")))
    }

    /// The shared `meta.db` connection, opened on first use.
    ///
    /// Lives as long as this handle, which is invalidated whenever the index files
    /// change on disk or an index run finishes, so it never outlives the index it
    /// was opened against. WAL readers coexist with a running indexer, and
    /// `open_meta_db` sets the busy timeout.
    pub fn meta_conn(&self) -> Result<std::sync::MutexGuard<'_, rusqlite::Connection>> {
        if self.meta.get().is_none() {
            let conn = crate::cache::open_meta_db(self.cache_dir.join(crate::cache::META_DB))
                .context("Failed to open meta.db")?;
            crate::symbol_cache::SymbolCache::ensure_schema(&conn)
                .context("Failed to initialise the symbol cache schema")?;
            // A concurrent first caller may have won the race; either connection is fine.
            let _ = self.meta.set(Mutex::new(conn));
        }
        let m = self.meta.get().expect("set above");
        Ok(m.lock().unwrap_or_else(|poisoned| poisoned.into_inner()))
    }

    /// Number of live files in the index.
    pub fn file_count(&self) -> usize {
        self.snapshot.live_file_count()
    }

    /// Store file id of a live file, by path as stored in the index (a leading
    /// `./` is ignored).
    pub fn file_id_for(&self, path: &str) -> Option<u32> {
        let normalized = path.strip_prefix("./").unwrap_or(path);
        let map = self.path_to_id.get_or_init(|| {
            let mut map = HashMap::with_capacity(self.snapshot.live_file_count());
            for id in self.snapshot.live_ids() {
                if let Some(p) = self.snapshot.get_file_path(id).and_then(|p| p.to_str()) {
                    map.entry(p.strip_prefix("./").unwrap_or(p).to_string())
                        .or_insert(id);
                }
            }
            map
        });
        map.get(normalized).copied()
    }

    /// Path for a store file id, without a leading `./`.
    pub fn path_of(&self, file_id: u32) -> Option<&str> {
        self.snapshot
            .get_file_path(file_id)
            .and_then(|p| p.to_str())
            .map(|p| p.strip_prefix("./").unwrap_or(p))
    }

    /// The query-side thread pool, built on first use.
    pub fn pool(&self) -> &rayon::ThreadPool {
        self.pool.get_or_init(|| {
            rayon::ThreadPoolBuilder::new()
                .num_threads(self.threads)
                .thread_name(|i| format!("rfx-query-{i}"))
                .build()
                .unwrap_or_else(|e| {
                    // A pool that cannot be built (thread limit hit) degrades to
                    // the global pool rather than failing the query.
                    log::warn!("Failed to create query thread pool: {}; using default", e);
                    rayon::ThreadPoolBuilder::new()
                        .num_threads(1)
                        .build()
                        .expect("a single-thread pool")
                })
        })
    }

    /// `max_posting_list_entries` the index was configured with (0 = unlimited).
    pub fn posting_cap(&self) -> usize {
        self.posting_cap
    }

    /// The cache directory this handle was opened from.
    pub fn cache_dir(&self) -> &Path {
        &self.cache_dir
    }
}

fn registry() -> &'static Mutex<HashMap<PathBuf, Arc<OpenIndex>>> {
    static REGISTRY: OnceLock<Mutex<HashMap<PathBuf, Arc<OpenIndex>>>> = OnceLock::new();
    REGISTRY.get_or_init(|| Mutex::new(HashMap::new()))
}

fn registry_key(cache_dir: &Path) -> PathBuf {
    cache_dir
        .canonicalize()
        .unwrap_or_else(|_| cache_dir.to_path_buf())
}

/// The shared handle for `cache`, opening it if no current one exists.
///
/// Cost on a hit: one lock, two `stat` calls, one `canonicalize`. A handle whose
/// files have been replaced on disk is dropped and reopened.
pub fn get_or_open(cache: &CacheManager) -> Result<Arc<OpenIndex>> {
    let cache_dir = cache.path();
    let key = registry_key(cache_dir);

    let fingerprint = match Fingerprint::current(cache_dir) {
        Ok(fp) => fp,
        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
            return Err(ReflexError::CacheCorrupted(format!(
                "content.bin: missing from {}",
                cache_dir.display()
            ))
            .into());
        }
        Err(e) => return Err(e).context("Failed to stat index files"),
    };

    if let Ok(map) = registry().lock()
        && let Some(existing) = map.get(&key)
        && existing.fingerprint == fingerprint
    {
        return Ok(Arc::clone(existing));
    }

    let opened = Arc::new(OpenIndex::open(cache, fingerprint)?);
    if let Ok(mut map) = registry().lock() {
        map.insert(key, Arc::clone(&opened));
    }
    Ok(opened)
}

/// Drop the registry entry for `cache_dir`, so the next lookup reopens.
///
/// Called before and after an index write; also safe to call when nothing is open.
pub fn invalidate(cache_dir: &Path) {
    let key = registry_key(cache_dir);
    if let Ok(mut map) = registry().lock() {
        map.remove(&key);
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::indexer::Indexer;
    use crate::models::IndexConfig;
    use tempfile::TempDir;

    fn indexed_project() -> TempDir {
        let temp = TempDir::new().unwrap();
        std::fs::write(temp.path().join("a.rs"), "fn alpha() {}\n").unwrap();
        std::fs::write(temp.path().join("b.rs"), "fn beta() { alpha() }\n").unwrap();
        Indexer::new(CacheManager::new(temp.path()), IndexConfig::default())
            .index(temp.path(), false)
            .unwrap();
        temp
    }

    #[test]
    fn second_lookup_returns_same_handle() {
        let temp = indexed_project();
        let cache = CacheManager::new(temp.path());
        let first = get_or_open(&cache).unwrap();
        let second = get_or_open(&cache).unwrap();
        assert!(Arc::ptr_eq(&first, &second));
    }

    #[test]
    fn file_id_lookup_ignores_dot_slash() {
        let temp = indexed_project();
        let open = get_or_open(&CacheManager::new(temp.path())).unwrap();
        let id = open.file_id_for("a.rs").expect("a.rs is indexed");
        assert_eq!(open.file_id_for("./a.rs"), Some(id));
        assert_eq!(open.path_of(id), Some("a.rs"));
        assert_eq!(open.file_id_for("missing.rs"), None);
    }

    #[test]
    fn reindex_yields_new_handle() {
        let temp = indexed_project();
        let cache = CacheManager::new(temp.path());
        let first = get_or_open(&cache).unwrap();
        assert_eq!(first.file_count(), 2);

        std::fs::write(temp.path().join("c.rs"), "fn gamma() {}\n").unwrap();
        Indexer::new(CacheManager::new(temp.path()), IndexConfig::default())
            .index(temp.path(), false)
            .unwrap();

        let second = get_or_open(&cache).unwrap();
        assert!(!Arc::ptr_eq(&first, &second));
        assert_eq!(second.file_count(), 3);
    }

    #[test]
    fn truncated_content_bin_is_reported_as_corruption() {
        let temp = indexed_project();
        let content = temp.path().join(".reflex/content.bin");
        std::fs::OpenOptions::new()
            .write(true)
            .open(&content)
            .unwrap()
            .set_len(2)
            .unwrap();

        let err = get_or_open(&CacheManager::new(temp.path())).unwrap_err();
        let typed = err
            .downcast_ref::<ReflexError>()
            .expect("typed CacheCorrupted");
        assert!(matches!(typed, ReflexError::CacheCorrupted(_)));
        assert!(err.to_string().contains("content.bin"), "{err}");
    }
}