Skip to main content

reflex/
symbol_cache.rs

1//! Symbol cache for storing parsed symbols
2//!
3//! This module provides transparent caching of parsed symbols to avoid
4//! re-parsing files during symbol queries. Symbols are stored in SQLite
5//! and keyed by (file_path, blake3_hash) for automatic invalidation when
6//! files change.
7
8use anyhow::{Context, Result};
9use rusqlite::OptionalExtension;
10use std::path::Path;
11
12use crate::models::SearchResult;
13
14#[cfg(test)]
15use crate::models::{Language, Span, SymbolKind};
16
17/// Symbol cache for storing and retrieving parsed symbols
18pub struct SymbolCache {
19    db_path: std::path::PathBuf,
20}
21
22/// Version of the cached symbol payload format.
23///
24/// Bump whenever the SHAPE or SIZE of a cached `SearchResult` changes, so existing
25/// caches are dropped rather than served stale.
26///
27/// * v1 — pre-1.7.2: previews bounded in lines only, so a minified file cached
28///   multi-megabyte previews.
29/// * v2 — 1.7.2: previews bounded at `parsers::preview::PREVIEW_MAX_BYTES`.
30/// * v3 — 2.0.0: `symbols_json` holds [`encode_symbols`] output — zstd-compressed
31///   JSON behind a 4-byte magic (raw JSON below 256 bytes). ~5x smaller on disk and
32///   ~5x less to write; decoding a candidate file costs microseconds.
33const SYMBOL_FORMAT_VERSION: i64 = 3;
34
35/// Marks a zstd-compressed symbol blob. Raw JSON starts with `[`, so the two
36/// encodings can never be confused.
37const BLOB_MAGIC: [u8; 4] = [0xFF, b'R', b'Z', 0x01];
38/// Blobs shorter than this are stored as raw JSON; compression would not pay.
39const BLOB_COMPRESS_MIN: usize = 256;
40/// zstd level: fast, and this JSON is repetitive enough that higher levels gain little.
41const BLOB_ZSTD_LEVEL: i32 = 3;
42
43/// Serialize any value as a compact blob: zstd-compressed JSON behind [`BLOB_MAGIC`],
44/// or raw JSON when shorter than [`BLOB_COMPRESS_MIN`]. Pulse's API cache uses it too.
45pub fn encode_json_blob<T: serde::Serialize + ?Sized>(value: &T) -> Result<Vec<u8>> {
46    let json = serde_json::to_vec(value).context("Failed to serialize blob")?;
47    if json.len() < BLOB_COMPRESS_MIN {
48        return Ok(json);
49    }
50    let compressed =
51        zstd::bulk::compress(&json, BLOB_ZSTD_LEVEL).context("Failed to compress blob")?;
52    let mut out = Vec::with_capacity(BLOB_MAGIC.len() + compressed.len());
53    out.extend_from_slice(&BLOB_MAGIC);
54    out.extend_from_slice(&compressed);
55    Ok(out)
56}
57
58/// Deserialize a blob written by [`encode_json_blob`] (or raw JSON).
59pub fn decode_json_blob<T: serde::de::DeserializeOwned>(bytes: &[u8]) -> Result<T> {
60    if let Some(body) = bytes.strip_prefix(&BLOB_MAGIC) {
61        let json = zstd::decode_all(body).context("Failed to decompress blob")?;
62        return serde_json::from_slice(&json).context("Failed to deserialize blob");
63    }
64    if matches!(bytes.first(), Some(b'[') | Some(b'{')) {
65        return serde_json::from_slice(bytes).context("Failed to deserialize blob");
66    }
67    anyhow::bail!("Unrecognised blob encoding ({} bytes)", bytes.len())
68}
69
70/// Serialize symbols for the `symbols_json` column.
71pub fn encode_symbols(symbols: &[SearchResult]) -> Result<Vec<u8>> {
72    encode_json_blob(symbols).context("Failed to encode symbols")
73}
74
75/// Deserialize a `symbols_json` column value written by [`encode_symbols`] (or a
76/// raw JSON array).
77pub fn decode_symbols(bytes: &[u8]) -> Result<Vec<SearchResult>> {
78    if bytes.first() == Some(&b'{') {
79        anyhow::bail!("Unrecognised symbol blob encoding ({} bytes)", bytes.len());
80    }
81    decode_json_blob(bytes).context("Failed to deserialize cached symbols")
82}
83
84/// Read the `symbols_json` column at `idx`, whether stored as BLOB (v3) or TEXT.
85pub fn read_symbols_column(row: &rusqlite::Row<'_>, idx: usize) -> rusqlite::Result<Vec<u8>> {
86    use rusqlite::types::ValueRef;
87    match row.get_ref(idx)? {
88        ValueRef::Blob(b) | ValueRef::Text(b) => Ok(b.to_vec()),
89        other => Err(rusqlite::Error::InvalidColumnType(
90            idx,
91            "symbols_json".to_string(),
92            other.data_type(),
93        )),
94    }
95}
96
97impl SymbolCache {
98    /// Open a symbol cache at the given cache directory
99    pub fn open(cache_dir: &Path) -> Result<Self> {
100        let db_path = cache_dir.join("meta.db");
101
102        if !db_path.exists() {
103            anyhow::bail!("Cache not initialized - run 'rfx index' first");
104        }
105
106        let cache = Self { db_path };
107        cache.init_schema()?;
108
109        Ok(cache)
110    }
111
112    /// Initialize the symbols table schema if it doesn't exist
113    fn init_schema(&self) -> Result<()> {
114        let conn = crate::cache::open_meta_db(&self.db_path).context("Failed to open meta.db")?;
115        Self::ensure_schema(&conn)
116    }
117
118    /// Create or migrate the `symbols` table on an already-open connection.
119    ///
120    /// The query path calls this once per index handle (see
121    /// `OpenIndex::meta_conn`), not once per query: it is several statements
122    /// (`pragma_table_info`, `CREATE TABLE`, two `CREATE INDEX`, a version read)
123    /// that cost real milliseconds and never change between queries.
124    pub fn ensure_schema(conn: &rusqlite::Connection) -> Result<()> {
125        // Check if we need to migrate to file_id-based schema
126        let uses_file_id: bool = conn
127            .query_row(
128                "SELECT COUNT(*) FROM pragma_table_info('symbols') WHERE name='file_id'",
129                [],
130                |row| row.get::<_, i64>(0),
131            )
132            .unwrap_or(0)
133            > 0;
134
135        if !uses_file_id {
136            // Old schema detected - drop and recreate with new schema
137            let table_exists: bool = conn
138                .query_row(
139                    "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='symbols'",
140                    [],
141                    |row| row.get::<_, i64>(0),
142                )
143                .unwrap_or(0)
144                > 0;
145
146            if table_exists {
147                log::warn!("Symbol cache schema outdated - migrating to file_id-based schema");
148                conn.execute("DROP TABLE IF EXISTS symbols", [])?;
149            }
150        }
151
152        // Create symbols table with file_id instead of file_path
153        conn.execute(
154            "CREATE TABLE IF NOT EXISTS symbols (
155                file_id INTEGER NOT NULL,
156                file_hash TEXT NOT NULL,
157                symbols_json TEXT NOT NULL,
158                last_cached INTEGER NOT NULL,
159                PRIMARY KEY (file_id, file_hash),
160                FOREIGN KEY (file_id) REFERENCES files(id) ON DELETE CASCADE
161            )",
162            [],
163        )?;
164
165        conn.execute(
166            "CREATE INDEX IF NOT EXISTS idx_symbols_file_id ON symbols(file_id)",
167            [],
168        )?;
169
170        conn.execute(
171            "CREATE INDEX IF NOT EXISTS idx_symbols_hash ON symbols(file_hash)",
172            [],
173        )?;
174
175        // Invalidate caches written in an older PREVIEW format.
176        //
177        // `src/parsers/` is not in `CACHE_CRITICAL_FILES` (build.rs), so changing
178        // preview extraction does not move `CACHE_SCHEMA_HASH` — and the symbol cache
179        // keys on file CONTENT hash, which does not change either. Without this, every
180        // cache written before 1.7.2 would keep its multi-megabyte previews forever.
181        //
182        // Deliberately NOT solved by adding the parsers to CACHE_CRITICAL_FILES: that
183        // would invalidate content.bin and the trigram index for a preview-format
184        // change, forcing a full reindex on every user — and since the version guard
185        // landed, a hash mismatch makes writers refuse outright.
186        let stored_format: Option<i64> = conn
187            .query_row(
188                "SELECT CAST(value AS INTEGER) FROM statistics WHERE key = 'symbol_format_version'",
189                [],
190                |row| row.get(0),
191            )
192            .optional()?;
193
194        if stored_format != Some(SYMBOL_FORMAT_VERSION) {
195            let dropped = conn.execute("DELETE FROM symbols", []).unwrap_or(0);
196            if dropped > 0 {
197                log::info!(
198                    "Symbol preview format changed (cache v{:?} -> v{}); cleared {} cached entries",
199                    stored_format,
200                    SYMBOL_FORMAT_VERSION,
201                    dropped
202                );
203            }
204            conn.execute(
205                "INSERT OR REPLACE INTO statistics (key, value, updated_at) VALUES (?, ?, ?)",
206                rusqlite::params![
207                    "symbol_format_version",
208                    SYMBOL_FORMAT_VERSION.to_string(),
209                    chrono::Utc::now().timestamp()
210                ],
211            )?;
212        }
213
214        log::debug!("Symbol cache schema initialized (file_id-based)");
215        Ok(())
216    }
217
218    /// Get cached symbols for a file (returns None if not cached or hash mismatch)
219    pub fn get(&self, file_path: &str, file_hash: &str) -> Result<Option<Vec<SearchResult>>> {
220        let conn = crate::cache::open_meta_db(&self.db_path)?;
221
222        // Lookup file_id
223        let file_id: Option<i64> = conn
224            .query_row("SELECT id FROM files WHERE path = ?", [file_path], |row| {
225                row.get(0)
226            })
227            .optional()?;
228
229        let Some(file_id) = file_id else {
230            log::debug!("Symbol cache MISS: {} (file not in index)", file_path);
231            return Ok(None);
232        };
233
234        let symbols_blob: Option<Vec<u8>> = conn
235            .query_row(
236                "SELECT symbols_json FROM symbols WHERE file_id = ? AND file_hash = ?",
237                [&file_id.to_string(), file_hash],
238                |row| read_symbols_column(row, 0),
239            )
240            .optional()?;
241
242        match symbols_blob {
243            Some(blob) => {
244                let mut symbols: Vec<SearchResult> = decode_symbols(&blob)?;
245
246                // Restore file_path (it was removed during serialization to save space)
247                for symbol in &mut symbols {
248                    symbol.path = file_path.to_string();
249                    symbol.lang = crate::models::Language::from_path(Path::new(file_path));
250                }
251
252                log::debug!(
253                    "Symbol cache HIT: {} ({} symbols)",
254                    file_path,
255                    symbols.len()
256                );
257                Ok(Some(symbols))
258            }
259            None => {
260                log::debug!("Symbol cache MISS: {}", file_path);
261                Ok(None)
262            }
263        }
264    }
265
266    /// Get cached symbols for multiple files in one transaction (batch read)
267    ///
268    /// This is significantly faster than calling `get()` repeatedly because:
269    /// - Opens only ONE database connection instead of N
270    /// - Reuses ONE prepared statement instead of creating N
271    /// - Executes in ONE transaction instead of N
272    ///
273    /// Returns results in the same order as input. None means cache miss or hash mismatch.
274    pub fn batch_get(
275        &self,
276        files: &[(String, String)],
277    ) -> Result<Vec<(String, Option<Vec<SearchResult>>)>> {
278        if files.is_empty() {
279            return Ok(Vec::new());
280        }
281
282        let conn = crate::cache::open_meta_db(&self.db_path)?;
283
284        // Prepare statements for file_id lookup and symbol retrieval
285        let mut file_id_stmt = conn.prepare("SELECT id FROM files WHERE path = ?")?;
286        let mut symbols_stmt =
287            conn.prepare("SELECT symbols_json FROM symbols WHERE file_id = ? AND file_hash = ?")?;
288
289        let mut results = Vec::with_capacity(files.len());
290        let mut hits = 0;
291        let mut misses = 0;
292
293        for (file_path, file_hash) in files {
294            // Lookup file_id
295            let file_id: Option<i64> = file_id_stmt
296                .query_row([file_path.as_str()], |row| row.get(0))
297                .optional()?;
298
299            let symbols = if let Some(file_id) = file_id {
300                let symbols_blob: Option<Vec<u8>> = symbols_stmt
301                    .query_row([&file_id.to_string(), file_hash.as_str()], |row| {
302                        read_symbols_column(row, 0)
303                    })
304                    .optional()?;
305
306                match symbols_blob {
307                    Some(blob) => {
308                        match decode_symbols(&blob) {
309                            Ok(mut symbols) => {
310                                // Restore file_path (it was removed during serialization to save space)
311                                for symbol in &mut symbols {
312                                    symbol.path = file_path.clone();
313                                    symbol.lang =
314                                        crate::models::Language::from_path(Path::new(file_path));
315                                }
316                                hits += 1;
317                                Some(symbols)
318                            }
319                            Err(e) => {
320                                log::warn!(
321                                    "Failed to deserialize cached symbols for {}: {}",
322                                    file_path,
323                                    e
324                                );
325                                misses += 1;
326                                None
327                            }
328                        }
329                    }
330                    None => {
331                        misses += 1;
332                        None
333                    }
334                }
335            } else {
336                misses += 1;
337                None
338            };
339
340            results.push((file_path.clone(), symbols));
341        }
342
343        log::debug!(
344            "Batch symbol cache: {} hits, {} misses ({}  total)",
345            hits,
346            misses,
347            files.len()
348        );
349        Ok(results)
350    }
351
352    /// Get cached symbols for multiple files with optional kind filtering
353    ///
354    /// Uses integer file_ids for fast batch retrieval, then filters by kind in Rust.
355    /// This avoids the cache miss detection bug that occurs with SQL-level filtering.
356    ///
357    /// Automatically chunks large batches to avoid SQLite parameter limits (999 max).
358    ///
359    /// Parameters:
360    /// - file_ids: Vec of (file_id, file_hash, file_path) tuples
361    /// - kind_filter: Optional symbol kind to filter by (applied in Rust after retrieval)
362    ///
363    /// Returns HashMap of file_id → symbols for cache hits.
364    pub fn batch_get_with_kind(
365        &self,
366        file_ids: &[(i64, String, String)], // (file_id, hash, path)
367        kind_filter: Option<crate::models::SymbolKind>,
368    ) -> Result<std::collections::HashMap<i64, Vec<SearchResult>>> {
369        if file_ids.is_empty() {
370            return Ok(std::collections::HashMap::new());
371        }
372        let conn = crate::cache::open_meta_db(&self.db_path)?;
373        Self::batch_get_with_kind_on(&conn, file_ids, kind_filter)
374    }
375
376    /// [`Self::batch_get_with_kind`] on an already-open connection.
377    ///
378    /// Only rows whose stored `file_hash` equals the expected hash are returned.
379    /// `INSERT OR REPLACE` keys on `(file_id, file_hash)`, so a file that changed
380    /// keeps its old-hash row next to the new one; before 2.0.0 this read ignored
381    /// the hash and could serve a changed file its pre-change symbols.
382    pub fn batch_get_with_kind_on(
383        conn: &rusqlite::Connection,
384        file_ids: &[(i64, String, String)], // (file_id, hash, path)
385        kind_filter: Option<crate::models::SymbolKind>,
386    ) -> Result<std::collections::HashMap<i64, Vec<SearchResult>>> {
387        use std::collections::HashMap;
388
389        if file_ids.is_empty() {
390            return Ok(HashMap::new());
391        }
392
393        // SQLite has a limit of 999 parameters by default
394        // Chunk requests to stay well under that limit
395        const BATCH_SIZE: usize = 900;
396
397        // Build lookup map for file_ids → (hash, path)
398        let file_info: HashMap<i64, (String, String)> = file_ids
399            .iter()
400            .map(|(id, hash, path)| (*id, (hash.clone(), path.clone())))
401            .collect();
402
403        // Capture kind filter for Rust-side filtering
404        let kind_for_filtering = kind_filter.clone();
405
406        // Collect results across all chunks
407        let mut cache_map: HashMap<i64, Vec<SearchResult>> = HashMap::new();
408        let mut hits = 0;
409
410        for chunk in file_ids.chunks(BATCH_SIZE) {
411            // Build placeholders for IN clause for this chunk
412            let id_placeholders = chunk.iter().map(|_| "?").collect::<Vec<_>>().join(", ");
413
414            // Always use simple query - filter by kind in Rust to avoid cache miss detection bug
415            let query = format!(
416                "SELECT file_id, file_hash, symbols_json
417                 FROM symbols
418                 WHERE file_id IN ({})",
419                id_placeholders
420            );
421
422            // Prepare parameters for this chunk
423            let params: Vec<Box<dyn rusqlite::ToSql>> = chunk
424                .iter()
425                .map(|(id, _, _)| Box::new(*id) as Box<dyn rusqlite::ToSql>)
426                .collect();
427
428            // Execute query
429            let mut stmt = conn.prepare(&query)?;
430            let param_refs: Vec<&dyn rusqlite::ToSql> = params.iter().map(|p| p.as_ref()).collect();
431            let rows = stmt.query_map(param_refs.as_slice(), |row| {
432                Ok((
433                    row.get::<_, i64>(0)?,
434                    row.get::<_, String>(1)?,
435                    read_symbols_column(row, 2)?,
436                ))
437            })?;
438
439            for row_result in rows {
440                let (file_id, stored_hash, symbols_blob) = row_result?;
441
442                // Only the row for the CURRENT content counts as a hit.
443                if let Some((hash, file_path)) = file_info.get(&file_id) {
444                    if *hash != stored_hash {
445                        continue;
446                    }
447                    match decode_symbols(&symbols_blob) {
448                        Ok(mut symbols) => {
449                            // Restore file_path (it was removed during serialization). `lang` is
450                            // `#[serde(skip)]`, so it must be re-derived from the path too, or every
451                            // cached symbol comes back as the default language.
452                            for symbol in &mut symbols {
453                                symbol.path = file_path.clone();
454                                symbol.lang =
455                                    crate::models::Language::from_path(Path::new(file_path));
456                            }
457
458                            // Filter symbols by kind if needed (Rust-side filtering)
459                            // Note: We do this in Rust rather than SQL to avoid cache miss detection bugs
460                            // SQL filtering would exclude files without the kind, making QueryEngine think they're uncached
461                            if let Some(ref filter_kind) = kind_for_filtering {
462                                symbols.retain(|s| &s.kind == filter_kind);
463                            }
464
465                            cache_map.insert(file_id, symbols);
466                            hits += 1;
467                        }
468                        Err(e) => {
469                            log::warn!(
470                                "Failed to deserialize cached symbols for file_id {}: {}",
471                                file_id,
472                                e
473                            );
474                        }
475                    }
476                }
477            }
478        }
479
480        let misses = file_ids.len() - hits;
481
482        if kind_for_filtering.is_some() {
483            log::debug!(
484                "Batch symbol cache with Rust-side kind filter: {} hits, {} misses ({} total, {} chunks)",
485                hits,
486                misses,
487                file_ids.len(),
488                file_ids.len().div_ceil(BATCH_SIZE)
489            );
490        } else {
491            log::debug!(
492                "Batch symbol cache: {} hits, {} misses ({} total, {} chunks)",
493                hits,
494                misses,
495                file_ids.len(),
496                file_ids.len().div_ceil(BATCH_SIZE)
497            );
498        }
499
500        Ok(cache_map)
501    }
502
503    /// Store symbols for a file using file_id
504    pub fn set(&self, file_path: &str, file_hash: &str, symbols: &[SearchResult]) -> Result<()> {
505        let conn = crate::cache::open_meta_db(&self.db_path)?;
506
507        // Lookup file_id from file_path
508        let file_id: i64 = conn
509            .query_row("SELECT id FROM files WHERE path = ?", [file_path], |row| {
510                row.get(0)
511            })
512            .context(format!("File not found in index: {}", file_path))?;
513
514        // Serialize symbols WITHOUT path (we'll restore it on read to save ~90MB)
515        let symbols_without_path: Vec<_> = symbols
516            .iter()
517            .map(|s| {
518                let mut s = s.clone();
519                s.path = String::new(); // Clear path to avoid duplication
520                s
521            })
522            .collect();
523
524        let symbols_blob = encode_symbols(&symbols_without_path)?;
525
526        let now = chrono::Utc::now().timestamp();
527
528        conn.execute(
529            "INSERT OR REPLACE INTO symbols (file_id, file_hash, symbols_json, last_cached)
530             VALUES (?, ?, ?, ?)",
531            rusqlite::params![file_id, file_hash, symbols_blob, now.to_string()],
532        )?;
533
534        log::debug!("Cached {} symbols for {}", symbols.len(), file_path);
535        Ok(())
536    }
537
538    /// Store parsed symbols for files already resolved to ids, in one transaction
539    /// on an open connection. Used by the query path for cache misses, which
540    /// previously opened a connection and ran an `INSERT` per file from inside the
541    /// parse pool.
542    pub fn batch_set_by_id_on(
543        conn: &mut rusqlite::Connection,
544        entries: &[(i64, String, Vec<SearchResult>)],
545    ) -> Result<()> {
546        if entries.is_empty() {
547            return Ok(());
548        }
549        let tx = conn.transaction()?;
550        let now_str = chrono::Utc::now().timestamp().to_string();
551        {
552            let mut stmt = tx.prepare(
553                "INSERT OR REPLACE INTO symbols (file_id, file_hash, symbols_json, last_cached)
554                 VALUES (?, ?, ?, ?)",
555            )?;
556            for (file_id, file_hash, symbols) in entries {
557                let symbols_blob = encode_symbols(symbols)?;
558                stmt.execute(rusqlite::params![file_id, file_hash, symbols_blob, now_str])?;
559            }
560        }
561        tx.commit()?;
562        log::debug!("Batch cached symbols for {} files (by id)", entries.len());
563        Ok(())
564    }
565
566    /// Batch store symbols for multiple files in a single transaction
567    pub fn batch_set(&self, entries: &[(String, String, Vec<SearchResult>)]) -> Result<()> {
568        let mut conn = crate::cache::open_meta_db(&self.db_path)?;
569        let tx = conn.transaction()?;
570
571        let now = chrono::Utc::now().timestamp();
572        let now_str = now.to_string();
573
574        for (file_path, file_hash, symbols) in entries {
575            // Lookup file_id
576            let file_id: i64 = tx
577                .query_row(
578                    "SELECT id FROM files WHERE path = ?",
579                    [file_path.as_str()],
580                    |row| row.get(0),
581                )
582                .context(format!("File not found in index: {}", file_path))?;
583
584            // Serialize symbols directly.
585            //
586            // This used to clone every SearchResult to blank its `path` — doubling
587            // peak memory for a batch, which on a minified bundle meant 18.7 GiB
588            // became 37.4 GiB. The clone was a no-op: all 55 construction sites in
589            // `src/parsers/` already pass `String::new()` as the path, and `get()`
590            // overwrites `path` unconditionally on read regardless.
591            // A caller that does set `path` costs a few bytes per symbol in the blob
592            // and nothing else, since `get()` replaces it with the real path on read.
593            // That is a far better trade than cloning the whole batch to save them.
594            let symbols_blob = encode_symbols(symbols)?;
595
596            // Insert into symbols table
597            tx.execute(
598                "INSERT OR REPLACE INTO symbols (file_id, file_hash, symbols_json, last_cached)
599                 VALUES (?, ?, ?, ?)",
600                rusqlite::params![file_id, file_hash.as_str(), symbols_blob, now_str],
601            )?;
602        }
603
604        tx.commit()?;
605        log::debug!("Batch cached symbols for {} files", entries.len());
606        Ok(())
607    }
608
609    /// Clear all cached symbols
610    pub fn clear(&self) -> Result<()> {
611        let conn = crate::cache::open_meta_db(&self.db_path)?;
612        conn.execute("DELETE FROM symbols", [])?;
613        log::info!("Cleared symbol cache");
614        Ok(())
615    }
616
617    /// Get cache statistics
618    pub fn stats(&self) -> Result<SymbolCacheStats> {
619        let conn = crate::cache::open_meta_db(&self.db_path)?;
620
621        let total_files: usize = conn
622            .query_row("SELECT COUNT(DISTINCT file_id) FROM symbols", [], |row| {
623                row.get(0)
624            })
625            .unwrap_or(0);
626
627        let total_entries: usize = conn
628            .query_row("SELECT COUNT(*) FROM symbols", [], |row| row.get(0))
629            .unwrap_or(0);
630
631        // Cache size on disk: the sum of the stored (v3: compressed) blob lengths.
632        let cache_size_bytes: u64 = conn
633            .query_row("SELECT SUM(LENGTH(symbols_json)) FROM symbols", [], |row| {
634                row.get(0)
635            })
636            .unwrap_or(0);
637
638        Ok(SymbolCacheStats {
639            total_files,
640            total_entries,
641            cache_size_bytes,
642        })
643    }
644
645    /// Every `(file_id, file_hash)` pair with a cached entry, in one query.
646    ///
647    /// The background pass used to ask [`get`](Self::get) once per file — a fresh
648    /// connection and two queries each, on one thread, for every file in the tree.
649    pub fn load_cached_keys_on(
650        conn: &rusqlite::Connection,
651    ) -> Result<std::collections::HashSet<(i64, String)>> {
652        let mut stmt = conn.prepare("SELECT file_id, file_hash FROM symbols")?;
653        let keys = stmt
654            .query_map([], |row| {
655                Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?))
656            })?
657            .collect::<std::result::Result<std::collections::HashSet<_>, _>>()
658            .context("Failed to read cached symbol keys")?;
659        Ok(keys)
660    }
661
662    /// The `INSERT` the background pass's writer prepares once.
663    pub const INSERT_SYMBOLS_SQL: &'static str = "INSERT OR REPLACE INTO symbols (file_id, file_hash, symbols_json, last_cached) \
664         VALUES (?, ?, ?, ?)";
665
666    /// [`cleanup_stale`](Self::cleanup_stale) on an open connection.
667    pub fn cleanup_stale_on(conn: &rusqlite::Connection) -> Result<usize> {
668        let removed = conn.execute(Self::CLEANUP_STALE_SQL, [])?;
669        if removed > 0 {
670            log::info!("Removed {} stale symbol cache entries", removed);
671        }
672        Ok(removed)
673    }
674
675    /// Rows no current file or branch version needs: a file that is gone, or a
676    /// version (hash) that is neither the stored one (`files.hash`) nor any
677    /// branch's (`file_branches.hash`). Since ids are stable across reindexing,
678    /// an edited file keeps its old-hash row until this runs; other branches'
679    /// versions stay cached for a checkout back.
680    const CLEANUP_STALE_SQL: &'static str = "DELETE FROM symbols
681         WHERE NOT EXISTS (
682                   SELECT 1 FROM files f
683                   WHERE f.id = symbols.file_id AND f.hash = symbols.file_hash)
684           AND NOT EXISTS (
685                   SELECT 1 FROM file_branches fb
686                   WHERE fb.file_id = symbols.file_id AND fb.hash = symbols.file_hash)";
687
688    /// Remove symbols no indexed file or branch version needs any more
689    /// (see [`CLEANUP_STALE_SQL`](Self::CLEANUP_STALE_SQL)).
690    pub fn cleanup_stale(&self) -> Result<usize> {
691        let conn = crate::cache::open_meta_db(&self.db_path)?;
692
693        let removed = conn.execute(Self::CLEANUP_STALE_SQL, [])?;
694
695        if removed > 0 {
696            log::info!("Removed {} stale symbol cache entries", removed);
697        }
698
699        Ok(removed)
700    }
701}
702
703/// Statistics about the symbol cache
704#[derive(Debug, Clone)]
705pub struct SymbolCacheStats {
706    pub total_files: usize,
707    pub total_entries: usize,
708    pub cache_size_bytes: u64,
709}
710
711#[cfg(test)]
712mod tests {
713    use super::*;
714    use crate::cache::CacheManager;
715    use tempfile::TempDir;
716
717    #[test]
718    fn test_symbol_cache_init() {
719        let temp = TempDir::new().unwrap();
720        let cache_mgr = CacheManager::new(temp.path());
721        cache_mgr.init().unwrap();
722
723        let symbol_cache = SymbolCache::open(cache_mgr.path()).unwrap();
724        let stats = symbol_cache.stats().unwrap();
725        assert_eq!(stats.total_files, 0);
726    }
727
728    #[test]
729    fn test_symbol_cache_set_get() {
730        let temp = TempDir::new().unwrap();
731        let cache_mgr = CacheManager::new(temp.path());
732        cache_mgr.init().unwrap();
733
734        // Add file to index first (required for symbol_cache.set())
735        cache_mgr.update_file("test.rs", "rust", 100).unwrap();
736
737        let symbol_cache = SymbolCache::open(cache_mgr.path()).unwrap();
738
739        let symbols = vec![SearchResult::new(
740            "test.rs".to_string(),
741            Language::Rust,
742            SymbolKind::Function,
743            Some("test_fn".to_string()),
744            Span::new(1, 0, 5, 0),
745            None,
746            "fn test_fn() {}".to_string(),
747        )];
748
749        // Store symbols
750        symbol_cache.set("test.rs", "hash123", &symbols).unwrap();
751
752        // Retrieve symbols
753        let cached = symbol_cache.get("test.rs", "hash123").unwrap();
754        assert!(cached.is_some());
755        assert_eq!(cached.as_ref().unwrap().len(), 1);
756        assert_eq!(cached.unwrap()[0].symbol.as_deref(), Some("test_fn"));
757    }
758
759    #[test]
760    fn test_symbol_cache_hash_mismatch() {
761        let temp = TempDir::new().unwrap();
762        let cache_mgr = CacheManager::new(temp.path());
763        cache_mgr.init().unwrap();
764
765        // Add file to index first
766        cache_mgr.update_file("test.rs", "rust", 100).unwrap();
767
768        let symbol_cache = SymbolCache::open(cache_mgr.path()).unwrap();
769
770        let symbols = vec![SearchResult::new(
771            "test.rs".to_string(),
772            Language::Rust,
773            SymbolKind::Function,
774            Some("test_fn".to_string()),
775            Span::new(1, 0, 5, 0),
776            None,
777            "fn test_fn() {}".to_string(),
778        )];
779
780        // Store with hash123
781        symbol_cache.set("test.rs", "hash123", &symbols).unwrap();
782
783        // Try to retrieve with different hash - should return None
784        let cached = symbol_cache.get("test.rs", "hash456").unwrap();
785        assert!(cached.is_none());
786    }
787
788    #[test]
789    fn test_symbol_cache_batch_set() {
790        let temp = TempDir::new().unwrap();
791        let cache_mgr = CacheManager::new(temp.path());
792        cache_mgr.init().unwrap();
793
794        // Add files to index first
795        cache_mgr.update_file("file1.rs", "rust", 100).unwrap();
796        cache_mgr.update_file("file2.rs", "rust", 200).unwrap();
797
798        let symbol_cache = SymbolCache::open(cache_mgr.path()).unwrap();
799
800        let entries = vec![
801            (
802                "file1.rs".to_string(),
803                "hash1".to_string(),
804                vec![SearchResult::new(
805                    "file1.rs".to_string(),
806                    Language::Rust,
807                    SymbolKind::Function,
808                    Some("fn1".to_string()),
809                    Span::new(1, 0, 5, 0),
810                    None,
811                    "fn fn1() {}".to_string(),
812                )],
813            ),
814            (
815                "file2.rs".to_string(),
816                "hash2".to_string(),
817                vec![SearchResult::new(
818                    "file2.rs".to_string(),
819                    Language::Rust,
820                    SymbolKind::Function,
821                    Some("fn2".to_string()),
822                    Span::new(1, 0, 5, 0),
823                    None,
824                    "fn fn2() {}".to_string(),
825                )],
826            ),
827        ];
828
829        symbol_cache.batch_set(&entries).unwrap();
830
831        let stats = symbol_cache.stats().unwrap();
832        assert_eq!(stats.total_files, 2);
833
834        let cached1 = symbol_cache.get("file1.rs", "hash1").unwrap();
835        assert!(cached1.is_some());
836
837        let cached2 = symbol_cache.get("file2.rs", "hash2").unwrap();
838        assert!(cached2.is_some());
839    }
840
841    #[test]
842    fn test_symbol_cache_batch_get() {
843        let temp = TempDir::new().unwrap();
844        let cache_mgr = CacheManager::new(temp.path());
845        cache_mgr.init().unwrap();
846
847        // Add files to index first
848        cache_mgr.update_file("file1.rs", "rust", 100).unwrap();
849        cache_mgr.update_file("file2.rs", "rust", 200).unwrap();
850        cache_mgr.update_file("file3.rs", "rust", 300).unwrap();
851
852        let symbol_cache = SymbolCache::open(cache_mgr.path()).unwrap();
853
854        // Populate cache with multiple files
855        let entries = vec![
856            (
857                "file1.rs".to_string(),
858                "hash1".to_string(),
859                vec![SearchResult::new(
860                    "file1.rs".to_string(),
861                    Language::Rust,
862                    SymbolKind::Function,
863                    Some("fn1".to_string()),
864                    Span::new(1, 0, 5, 0),
865                    None,
866                    "fn fn1() {}".to_string(),
867                )],
868            ),
869            (
870                "file2.rs".to_string(),
871                "hash2".to_string(),
872                vec![SearchResult::new(
873                    "file2.rs".to_string(),
874                    Language::Rust,
875                    SymbolKind::Struct,
876                    Some("Struct2".to_string()),
877                    Span::new(1, 0, 5, 0),
878                    None,
879                    "struct Struct2 {}".to_string(),
880                )],
881            ),
882            (
883                "file3.rs".to_string(),
884                "hash3".to_string(),
885                vec![SearchResult::new(
886                    "file3.rs".to_string(),
887                    Language::Rust,
888                    SymbolKind::Enum,
889                    Some("Enum3".to_string()),
890                    Span::new(1, 0, 5, 0),
891                    None,
892                    "enum Enum3 {}".to_string(),
893                )],
894            ),
895        ];
896
897        symbol_cache.batch_set(&entries).unwrap();
898
899        // Test batch_get with all cached files
900        let lookup = vec![
901            ("file1.rs".to_string(), "hash1".to_string()),
902            ("file2.rs".to_string(), "hash2".to_string()),
903            ("file3.rs".to_string(), "hash3".to_string()),
904        ];
905
906        let results = symbol_cache.batch_get(&lookup).unwrap();
907        assert_eq!(results.len(), 3);
908
909        // Verify all hits
910        assert!(results[0].1.is_some());
911        assert_eq!(
912            results[0].1.as_ref().unwrap()[0].symbol.as_deref(),
913            Some("fn1")
914        );
915
916        assert!(results[1].1.is_some());
917        assert_eq!(
918            results[1].1.as_ref().unwrap()[0].symbol.as_deref(),
919            Some("Struct2")
920        );
921
922        assert!(results[2].1.is_some());
923        assert_eq!(
924            results[2].1.as_ref().unwrap()[0].symbol.as_deref(),
925            Some("Enum3")
926        );
927
928        // Test batch_get with mixed hits and misses
929        let mixed_lookup = vec![
930            ("file1.rs".to_string(), "hash1".to_string()), // Hit
931            ("nonexistent.rs".to_string(), "hash999".to_string()), // Miss (file doesn't exist)
932            ("file2.rs".to_string(), "wrong_hash".to_string()), // Miss (hash mismatch)
933            ("file3.rs".to_string(), "hash3".to_string()), // Hit
934        ];
935
936        let mixed_results = symbol_cache.batch_get(&mixed_lookup).unwrap();
937        assert_eq!(mixed_results.len(), 4);
938
939        assert!(mixed_results[0].1.is_some()); // file1.rs - hit
940        assert!(mixed_results[1].1.is_none()); // nonexistent.rs - miss
941        assert!(mixed_results[2].1.is_none()); // file2.rs wrong hash - miss
942        assert!(mixed_results[3].1.is_some()); // file3.rs - hit
943
944        // Test batch_get with empty input
945        let empty_results = symbol_cache.batch_get(&[]).unwrap();
946        assert_eq!(empty_results.len(), 0);
947    }
948
949    #[test]
950    fn test_symbol_cache_clear() {
951        let temp = TempDir::new().unwrap();
952        let cache_mgr = CacheManager::new(temp.path());
953        cache_mgr.init().unwrap();
954
955        // Add file to index first
956        cache_mgr.update_file("test.rs", "rust", 100).unwrap();
957
958        let symbol_cache = SymbolCache::open(cache_mgr.path()).unwrap();
959
960        let symbols = vec![SearchResult::new(
961            "test.rs".to_string(),
962            Language::Rust,
963            SymbolKind::Function,
964            Some("test_fn".to_string()),
965            Span::new(1, 0, 5, 0),
966            None,
967            "fn test_fn() {}".to_string(),
968        )];
969
970        symbol_cache.set("test.rs", "hash123", &symbols).unwrap();
971
972        let stats_before = symbol_cache.stats().unwrap();
973        assert_eq!(stats_before.total_files, 1);
974
975        symbol_cache.clear().unwrap();
976
977        let stats_after = symbol_cache.stats().unwrap();
978        assert_eq!(stats_after.total_files, 0);
979    }
980
981    #[test]
982    fn test_symbol_cache_cleanup_stale() {
983        let temp = TempDir::new().unwrap();
984        let cache_mgr = CacheManager::new(temp.path());
985        cache_mgr.init().unwrap();
986
987        // Add a file to the index
988        cache_mgr.update_file("exists.rs", "rust", 100).unwrap();
989        cache_mgr
990            .record_branch_file("exists.rs", "main", "hash1", None)
991            .unwrap();
992
993        // Add deleted.rs to index temporarily
994        cache_mgr.update_file("deleted.rs", "rust", 200).unwrap();
995
996        let symbol_cache = SymbolCache::open(cache_mgr.path()).unwrap();
997
998        // Cache symbols for both existing and non-existing files
999        let symbols = vec![SearchResult::new(
1000            "test.rs".to_string(),
1001            Language::Rust,
1002            SymbolKind::Function,
1003            Some("test_fn".to_string()),
1004            Span::new(1, 0, 5, 0),
1005            None,
1006            "fn test_fn() {}".to_string(),
1007        )];
1008
1009        symbol_cache.set("exists.rs", "hash1", &symbols).unwrap();
1010        symbol_cache.set("deleted.rs", "hash2", &symbols).unwrap();
1011
1012        let stats_before = symbol_cache.stats().unwrap();
1013        assert_eq!(stats_before.total_files, 2);
1014
1015        // Now remove "deleted.rs" from files table to make its symbol cache entry stale
1016        // Note: With CASCADE DELETE foreign key constraint, the symbol entry is automatically
1017        // removed when the file is deleted, so cleanup_stale() won't find anything to remove.
1018        let conn = crate::cache::open_meta_db(cache_mgr.path().join("meta.db")).unwrap();
1019        conn.execute("DELETE FROM files WHERE path = 'deleted.rs'", [])
1020            .unwrap();
1021
1022        // Cleanup stale entries (should find 0 because CASCADE DELETE already cleaned it up)
1023        let removed = symbol_cache.cleanup_stale().unwrap();
1024        assert_eq!(removed, 0); // CASCADE DELETE already removed it
1025
1026        let stats_after = symbol_cache.stats().unwrap();
1027        assert_eq!(stats_after.total_files, 1);
1028
1029        // exists.rs should still be cached
1030        let cached = symbol_cache.get("exists.rs", "hash1").unwrap();
1031        assert!(cached.is_some());
1032
1033        // deleted.rs should be gone
1034        let cached2 = symbol_cache.get("deleted.rs", "hash2").unwrap();
1035        assert!(cached2.is_none());
1036    }
1037
1038    #[test]
1039    fn encode_decode_round_trip_raw_and_compressed() {
1040        let small = vec![SearchResult::new(
1041            String::new(),
1042            Language::Rust,
1043            SymbolKind::Function,
1044            Some("f".to_string()),
1045            Span::new(1, 0, 1, 0),
1046            None,
1047            "fn f() {}".to_string(),
1048        )];
1049        let raw = encode_symbols(&small).unwrap();
1050        assert_eq!(raw.first(), Some(&b'['), "short blobs stay raw JSON");
1051        assert_eq!(decode_symbols(&raw).unwrap().len(), 1);
1052
1053        let big: Vec<SearchResult> = (0..200)
1054            .map(|i| {
1055                SearchResult::new(
1056                    String::new(),
1057                    Language::Rust,
1058                    SymbolKind::Function,
1059                    Some(format!("function_number_{i}")),
1060                    Span::new(i, 0, i + 3, 0),
1061                    None,
1062                    format!("fn function_number_{i}() {{\n    body {i}\n}}"),
1063                )
1064            })
1065            .collect();
1066        let json = serde_json::to_vec(&big).unwrap();
1067        let blob = encode_symbols(&big).unwrap();
1068        assert!(blob.starts_with(&BLOB_MAGIC), "long blobs are compressed");
1069        assert!(
1070            blob.len() < json.len() / 3,
1071            "{} vs {}",
1072            blob.len(),
1073            json.len()
1074        );
1075        let back = decode_symbols(&blob).unwrap();
1076        assert_eq!(serde_json::to_vec(&back).unwrap(), json);
1077
1078        // A legacy raw-JSON row still decodes; garbage does not.
1079        assert_eq!(decode_symbols(&json).unwrap().len(), 200);
1080        assert!(decode_symbols(b"\xFFRZ\x02nope").is_err());
1081        assert!(decode_symbols(b"nope").is_err());
1082    }
1083
1084    #[test]
1085    fn load_cached_keys_matches_rows() {
1086        let temp = TempDir::new().unwrap();
1087        let cache_mgr = CacheManager::new(temp.path());
1088        cache_mgr.init().unwrap();
1089        cache_mgr.update_file("a.rs", "rust", 10).unwrap();
1090        cache_mgr.update_file("b.rs", "rust", 10).unwrap();
1091        let symbol_cache = SymbolCache::open(cache_mgr.path()).unwrap();
1092        symbol_cache.set("a.rs", "h1", &[]).unwrap();
1093        symbol_cache.set("b.rs", "h2", &[]).unwrap();
1094
1095        let conn = crate::cache::open_meta_db(cache_mgr.path().join("meta.db")).unwrap();
1096        let keys = SymbolCache::load_cached_keys_on(&conn).unwrap();
1097        let a: i64 = conn
1098            .query_row("SELECT id FROM files WHERE path = 'a.rs'", [], |r| r.get(0))
1099            .unwrap();
1100        let b: i64 = conn
1101            .query_row("SELECT id FROM files WHERE path = 'b.rs'", [], |r| r.get(0))
1102            .unwrap();
1103        assert_eq!(keys.len(), 2);
1104        assert!(keys.contains(&(a, "h1".to_string())));
1105        assert!(keys.contains(&(b, "h2".to_string())));
1106        assert!(!keys.contains(&(a, "h2".to_string())));
1107    }
1108}