Skip to main content

reflex/pulse/extract/
api_cache.rs

1//! The API of every source file, extracted once per content hash.
2//!
3//! `.reflex/pulse/api.db` maps `(path, content hash, extractor version)` to a
4//! zstd-compressed [`ApiFile`]. A run extracts only files whose content changed since
5//! the last run, in parallel, reading source from `content.bin` (never the working
6//! tree), and writes the new rows in one transaction.
7
8use super::{Corpus, FileRole};
9use crate::cache::CacheManager;
10use crate::parsers::api::{self, ApiFile, EXTRACTOR_VERSION};
11use crate::symbol_cache::{decode_json_blob, encode_json_blob};
12use anyhow::{Context, Result};
13use rayon::prelude::*;
14use std::collections::{BTreeMap, HashMap};
15use std::time::{Duration, Instant};
16
17/// Extracted APIs, keyed by index into `Corpus::files`.
18#[derive(Debug, Default)]
19pub struct ApiIndex {
20    pub files: BTreeMap<usize, ApiFile>,
21    pub stats: ApiStats,
22}
23
24#[derive(Debug, Default, Clone)]
25pub struct ApiStats {
26    pub cached: usize,
27    pub extracted: usize,
28    pub failed: usize,
29    pub elapsed: Duration,
30}
31
32fn open(cache: &CacheManager) -> Result<rusqlite::Connection> {
33    let dir = cache.path().join("pulse");
34    std::fs::create_dir_all(&dir)?;
35    let conn = crate::cache::open_meta_db(dir.join("api.db"))?;
36    conn.execute_batch(
37        "CREATE TABLE IF NOT EXISTS api_files (
38            path TEXT PRIMARY KEY,
39            hash TEXT NOT NULL,
40            version INTEGER NOT NULL,
41            blob BLOB NOT NULL
42        )",
43    )?;
44    Ok(conn)
45}
46
47/// Load (and fill) the API cache for every source file with an extractor.
48pub fn load(cache: &CacheManager, corpus: &Corpus) -> Result<ApiIndex> {
49    let start = Instant::now();
50    let wanted: Vec<usize> = corpus
51        .with_role(FileRole::Source)
52        .filter(|(_, f)| api::has_extractor(f.language))
53        .map(|(i, _)| i)
54        .collect();
55    let mut index = ApiIndex::default();
56    if wanted.is_empty() {
57        return Ok(index);
58    }
59
60    let mut conn = open(cache)?;
61    let mut cached: HashMap<String, (String, i64, Vec<u8>)> = HashMap::new();
62    {
63        let mut stmt = conn.prepare("SELECT path, hash, version, blob FROM api_files")?;
64        let rows = stmt.query_map([], |r| {
65            Ok((
66                r.get::<_, String>(0)?,
67                (
68                    r.get::<_, String>(1)?,
69                    r.get::<_, i64>(2)?,
70                    r.get::<_, Vec<u8>>(3)?,
71                ),
72            ))
73        })?;
74        for row in rows {
75            let (path, v) = row?;
76            cached.insert(path, v);
77        }
78    }
79
80    let mut misses = Vec::new();
81    for &i in &wanted {
82        let f = &corpus.files[i];
83        let hit = cached.get(&f.path).and_then(|(hash, version, blob)| {
84            (hash == &f.hash && *version == EXTRACTOR_VERSION as i64)
85                .then(|| decode_json_blob::<ApiFile>(blob).ok())
86                .flatten()
87        });
88        match hit {
89            Some(api) => {
90                index.files.insert(i, api);
91                index.stats.cached += 1;
92            }
93            None => misses.push(i),
94        }
95    }
96
97    if !misses.is_empty() {
98        let reader =
99            crate::snapshot::IndexSnapshot::open(cache.path()).context("opening content.bin")?;
100        let ids: HashMap<&str, u32> = reader
101            .live_ids()
102            .filter_map(|id| {
103                let p = reader.get_file_path(id)?.to_str()?;
104                Some((p.strip_prefix("./").unwrap_or(p), id))
105            })
106            .collect();
107        let extracted: Vec<(usize, Option<ApiFile>)> = misses
108            .par_iter()
109            .map(|&i| {
110                let f = &corpus.files[i];
111                let api = ids
112                    .get(f.path.strip_prefix("./").unwrap_or(&f.path))
113                    .and_then(|&id| reader.get_file_content(id).ok())
114                    .and_then(|src| api::extract(f.language, src));
115                (i, api)
116            })
117            .collect();
118
119        let tx = conn.transaction()?;
120        {
121            let mut put = tx.prepare(
122                "INSERT OR REPLACE INTO api_files (path, hash, version, blob) VALUES (?1, ?2, ?3, ?4)",
123            )?;
124            for (i, api) in extracted {
125                let f = &corpus.files[i];
126                match api {
127                    Some(api) => {
128                        put.execute(rusqlite::params![
129                            f.path,
130                            f.hash,
131                            EXTRACTOR_VERSION as i64,
132                            encode_json_blob(&api)?
133                        ])?;
134                        index.files.insert(i, api);
135                        index.stats.extracted += 1;
136                    }
137                    None => index.stats.failed += 1,
138                }
139            }
140        }
141        // Rows for files no longer indexed.
142        let live: std::collections::HashSet<&str> = wanted
143            .iter()
144            .map(|&i| corpus.files[i].path.as_str())
145            .collect();
146        for path in cached.keys().filter(|p| !live.contains(p.as_str())) {
147            tx.execute("DELETE FROM api_files WHERE path = ?1", [path])?;
148        }
149        tx.commit()?;
150    }
151    index.stats.elapsed = start.elapsed();
152    log::info!(
153        "pulse api: {} cached, {} extracted, {} failed in {:?}",
154        index.stats.cached,
155        index.stats.extracted,
156        index.stats.failed,
157        index.stats.elapsed
158    );
159    Ok(index)
160}