Skip to main content

reflex/pulse/extract/
mod.rs

1//! Extraction: read what the docs builders need from the Reflex index, once.
2//!
3//! Everything comes from the index (`meta.db`, `content.bin`), never from the working
4//! tree, so the site describes exactly what was indexed and every number agrees with
5//! `rfx query`.
6
7pub mod api_cache;
8pub mod capabilities;
9pub mod cli;
10pub mod roles;
11pub mod surface;
12
13use crate::cache::CacheManager;
14use crate::models::Language;
15use anyhow::{Context, Result};
16pub use roles::FileRole;
17use std::collections::BTreeSet;
18use std::path::{Path, PathBuf};
19
20/// Display name of a language: `Rust`, `TypeScript`, `C#`.
21pub fn language_name(lang: Language) -> String {
22    let id = serde_json::to_value(lang)
23        .ok()
24        .and_then(|v| v.as_str().map(str::to_string))
25        .unwrap_or_else(|| "unknown".into());
26    match id.as_str() {
27        "typescript" => "TypeScript".into(),
28        "javascript" => "JavaScript".into(),
29        "php" => "PHP".into(),
30        "csharp" => "C#".into(),
31        "cpp" => "C++".into(),
32        other => {
33            let mut c = other.chars();
34            match c.next() {
35                Some(f) => f.to_uppercase().chain(c).collect(),
36                None => other.to_string(),
37            }
38        }
39    }
40}
41
42/// One indexed file.
43#[derive(Debug, Clone)]
44pub struct FileInfo {
45    pub path: String,
46    pub language: Language,
47    pub lines: u64,
48    pub role: FileRole,
49    /// Content hash from `meta.db` (blake3); keys the API cache.
50    pub hash: String,
51}
52
53/// Reads indexed file contents from the index stores.
54pub struct ContentAccess {
55    reader: Option<crate::snapshot::IndexSnapshot>,
56    ids: std::collections::HashMap<String, u32>,
57}
58
59impl ContentAccess {
60    pub fn open(cache: &CacheManager) -> Self {
61        let reader = crate::snapshot::IndexSnapshot::open(cache.path()).ok();
62        let mut ids = std::collections::HashMap::new();
63        if let Some(r) = &reader {
64            for id in r.live_ids() {
65                if let Some(p) = r.get_file_path(id).and_then(|p| p.to_str()) {
66                    ids.insert(p.strip_prefix("./").unwrap_or(p).to_string(), id);
67                }
68            }
69        }
70        Self { reader, ids }
71    }
72
73    pub fn read(&self, path: &str) -> Option<&str> {
74        let id = *self.ids.get(path)?;
75        self.reader.as_ref()?.get_file_content(id).ok()
76    }
77}
78
79/// A repository document read from the index.
80#[derive(Debug, Clone)]
81pub struct DocFile {
82    pub path: String,
83    pub content: String,
84}
85
86/// An import of a module outside the project (external or standard library).
87#[derive(Debug, Clone)]
88pub struct Import {
89    pub file: usize,
90    /// As written: `axum::Router`, `github.com/spf13/cobra`, `fastapi`.
91    pub path: String,
92    pub line: u32,
93}
94
95/// The index, as the docs builders see it.
96#[derive(Debug, Clone)]
97pub struct Corpus {
98    pub root: PathBuf,
99    /// Sorted by path.
100    pub files: Vec<FileInfo>,
101    /// Resolved file-level imports as indices into `files` (importer, imported), deduped.
102    pub edges: Vec<(usize, usize)>,
103    /// External and standard-library imports, in file order.
104    pub imports: Vec<Import>,
105    pub readme: Option<DocFile>,
106}
107
108impl Corpus {
109    pub fn load(cache: &CacheManager) -> Result<Self> {
110        let root = cache
111            .path()
112            .parent()
113            .map(Path::to_path_buf)
114            .unwrap_or_else(|| PathBuf::from("."));
115        let conn = crate::cache::open_meta_db(cache.path().join("meta.db"))
116            .context("opening meta.db (run `rfx index` first)")?;
117
118        let mut rows: Vec<(i64, FileInfo)> = conn
119            .prepare("SELECT id, path, line_count, hash FROM files")?
120            .query_map([], |r| {
121                let path: String = r.get(1)?;
122                Ok((
123                    r.get::<_, i64>(0)?,
124                    FileInfo {
125                        language: Language::from_path(Path::new(&path)),
126                        role: roles::classify(&path),
127                        lines: r.get::<_, i64>(2)?.max(0) as u64,
128                        hash: r.get(3)?,
129                        path,
130                    },
131                ))
132            })?
133            .collect::<rusqlite::Result<_>>()?;
134        rows.sort_by(|a, b| a.1.path.cmp(&b.1.path));
135        let index_of: std::collections::HashMap<i64, usize> = rows
136            .iter()
137            .enumerate()
138            .map(|(i, (id, _))| (*id, i))
139            .collect();
140
141        let mut edges: BTreeSet<(usize, usize)> = BTreeSet::new();
142        let mut stmt = conn.prepare("SELECT src, dst FROM import_edges")?;
143        let pairs = stmt.query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, i64>(1)?)))?;
144        for pair in pairs {
145            let (from, to) = pair?;
146            if let (Some(&a), Some(&b)) = (index_of.get(&from), index_of.get(&to))
147                && a != b
148            {
149                edges.insert((a, b));
150            }
151        }
152
153        let mut imports = Vec::new();
154        let mut stmt = conn.prepare(
155            "SELECT file_id, imported_path, line_number FROM file_dependencies
156             WHERE import_type IN ('external', 'stdlib')",
157        )?;
158        let rows_iter = stmt.query_map([], |r| {
159            Ok((
160                r.get::<_, i64>(0)?,
161                r.get::<_, String>(1)?,
162                r.get::<_, i64>(2)?,
163            ))
164        })?;
165        for row in rows_iter {
166            let (file_id, path, line) = row?;
167            if let Some(&file) = index_of.get(&file_id) {
168                imports.push(Import {
169                    file,
170                    path,
171                    line: line.max(0) as u32,
172                });
173            }
174        }
175        imports.sort_by(|a, b| (a.file, a.line, &a.path).cmp(&(b.file, b.line, &b.path)));
176
177        let files: Vec<FileInfo> = rows.into_iter().map(|(_, f)| f).collect();
178        let readme = Self::read_readme(cache, &files);
179        Ok(Self {
180            root,
181            files,
182            edges: edges.into_iter().collect(),
183            imports,
184            readme,
185        })
186    }
187
188    /// The root README, preferring `README.md`.
189    fn read_readme(cache: &CacheManager, files: &[FileInfo]) -> Option<DocFile> {
190        let candidates = [
191            "README.md",
192            "README.mdx",
193            "README.markdown",
194            "README.rst",
195            "README.txt",
196            "README",
197        ];
198        let path = candidates.iter().find_map(|c| {
199            files
200                .iter()
201                .find(|f| f.path.eq_ignore_ascii_case(c))
202                .map(|f| f.path.clone())
203        })?;
204        let reader = crate::snapshot::IndexSnapshot::open(cache.path()).ok()?;
205        let id = reader.get_file_id_by_path(&path)?;
206        let content = reader.get_file_content(id).ok()?.to_string();
207        Some(DocFile { path, content })
208    }
209
210    /// Index into `files` for a path.
211    pub fn index_of(&self, path: &str) -> Option<usize> {
212        self.files
213            .binary_search_by(|f| f.path.as_str().cmp(path))
214            .ok()
215    }
216
217    /// Files with a given role.
218    pub fn with_role(&self, role: FileRole) -> impl Iterator<Item = (usize, &FileInfo)> {
219        self.files
220            .iter()
221            .enumerate()
222            .filter(move |(_, f)| f.role == role)
223    }
224
225    /// Count of indexed files per role.
226    pub fn role_counts(&self) -> std::collections::BTreeMap<String, usize> {
227        let mut out = std::collections::BTreeMap::new();
228        for f in &self.files {
229            *out.entry(f.role.as_str().to_string()).or_insert(0) += 1;
230        }
231        out
232    }
233}