reflex/pulse/extract/
api_cache.rs1use super::{Corpus, FileRole};
9use crate::cache::CacheManager;
10use crate::parsers::api::{self, ApiFile, EXTRACTOR_VERSION};
11use crate::symbol_cache::{decode_json_blob, encode_json_blob};
12use anyhow::{Context, Result};
13use rayon::prelude::*;
14use std::collections::{BTreeMap, HashMap};
15use std::time::{Duration, Instant};
16
17#[derive(Debug, Default)]
19pub struct ApiIndex {
20 pub files: BTreeMap<usize, ApiFile>,
21 pub stats: ApiStats,
22}
23
24#[derive(Debug, Default, Clone)]
25pub struct ApiStats {
26 pub cached: usize,
27 pub extracted: usize,
28 pub failed: usize,
29 pub elapsed: Duration,
30}
31
32fn open(cache: &CacheManager) -> Result<rusqlite::Connection> {
33 let dir = cache.path().join("pulse");
34 std::fs::create_dir_all(&dir)?;
35 let conn = crate::cache::open_meta_db(dir.join("api.db"))?;
36 conn.execute_batch(
37 "CREATE TABLE IF NOT EXISTS api_files (
38 path TEXT PRIMARY KEY,
39 hash TEXT NOT NULL,
40 version INTEGER NOT NULL,
41 blob BLOB NOT NULL
42 )",
43 )?;
44 Ok(conn)
45}
46
47pub fn load(cache: &CacheManager, corpus: &Corpus) -> Result<ApiIndex> {
49 let start = Instant::now();
50 let wanted: Vec<usize> = corpus
51 .with_role(FileRole::Source)
52 .filter(|(_, f)| api::has_extractor(f.language))
53 .map(|(i, _)| i)
54 .collect();
55 let mut index = ApiIndex::default();
56 if wanted.is_empty() {
57 return Ok(index);
58 }
59
60 let mut conn = open(cache)?;
61 let mut cached: HashMap<String, (String, i64, Vec<u8>)> = HashMap::new();
62 {
63 let mut stmt = conn.prepare("SELECT path, hash, version, blob FROM api_files")?;
64 let rows = stmt.query_map([], |r| {
65 Ok((
66 r.get::<_, String>(0)?,
67 (
68 r.get::<_, String>(1)?,
69 r.get::<_, i64>(2)?,
70 r.get::<_, Vec<u8>>(3)?,
71 ),
72 ))
73 })?;
74 for row in rows {
75 let (path, v) = row?;
76 cached.insert(path, v);
77 }
78 }
79
80 let mut misses = Vec::new();
81 for &i in &wanted {
82 let f = &corpus.files[i];
83 let hit = cached.get(&f.path).and_then(|(hash, version, blob)| {
84 (hash == &f.hash && *version == EXTRACTOR_VERSION as i64)
85 .then(|| decode_json_blob::<ApiFile>(blob).ok())
86 .flatten()
87 });
88 match hit {
89 Some(api) => {
90 index.files.insert(i, api);
91 index.stats.cached += 1;
92 }
93 None => misses.push(i),
94 }
95 }
96
97 if !misses.is_empty() {
98 let reader =
99 crate::snapshot::IndexSnapshot::open(cache.path()).context("opening content.bin")?;
100 let ids: HashMap<&str, u32> = reader
101 .live_ids()
102 .filter_map(|id| {
103 let p = reader.get_file_path(id)?.to_str()?;
104 Some((p.strip_prefix("./").unwrap_or(p), id))
105 })
106 .collect();
107 let extracted: Vec<(usize, Option<ApiFile>)> = misses
108 .par_iter()
109 .map(|&i| {
110 let f = &corpus.files[i];
111 let api = ids
112 .get(f.path.strip_prefix("./").unwrap_or(&f.path))
113 .and_then(|&id| reader.get_file_content(id).ok())
114 .and_then(|src| api::extract(f.language, src));
115 (i, api)
116 })
117 .collect();
118
119 let tx = conn.transaction()?;
120 {
121 let mut put = tx.prepare(
122 "INSERT OR REPLACE INTO api_files (path, hash, version, blob) VALUES (?1, ?2, ?3, ?4)",
123 )?;
124 for (i, api) in extracted {
125 let f = &corpus.files[i];
126 match api {
127 Some(api) => {
128 put.execute(rusqlite::params![
129 f.path,
130 f.hash,
131 EXTRACTOR_VERSION as i64,
132 encode_json_blob(&api)?
133 ])?;
134 index.files.insert(i, api);
135 index.stats.extracted += 1;
136 }
137 None => index.stats.failed += 1,
138 }
139 }
140 }
141 let live: std::collections::HashSet<&str> = wanted
143 .iter()
144 .map(|&i| corpus.files[i].path.as_str())
145 .collect();
146 for path in cached.keys().filter(|p| !live.contains(p.as_str())) {
147 tx.execute("DELETE FROM api_files WHERE path = ?1", [path])?;
148 }
149 tx.commit()?;
150 }
151 index.stats.elapsed = start.elapsed();
152 log::info!(
153 "pulse api: {} cached, {} extracted, {} failed in {:?}",
154 index.stats.cached,
155 index.stats.extracted,
156 index.stats.failed,
157 index.stats.elapsed
158 );
159 Ok(index)
160}