pub mod api;
pub mod c;
pub mod cpp;
pub mod csharp;
pub mod go;
pub mod java;
pub mod kotlin;
pub mod php;
pub mod preview;
pub mod python;
pub mod ruby;
pub mod rust;
pub mod svelte;
pub mod tsconfig;
pub mod typescript;
pub mod vue;
pub mod swift_imports;
pub mod zig;
use crate::models::{Language, SearchResult};
use anyhow::{Result, anyhow};
pub struct ParserFactory;
#[derive(Debug, Clone)]
pub struct ImportInfo {
pub imported_path: String,
pub import_type: crate::models::ImportType,
pub line_number: usize,
pub imported_symbols: Option<Vec<String>>,
}
#[derive(Debug, Clone)]
pub struct ExportInfo {
pub exported_symbol: Option<String>,
pub source_path: String,
pub line_number: usize,
}
pub trait DependencyExtractor {
fn extract_dependencies(source: &str) -> Result<Vec<ImportInfo>>;
}
impl ParserFactory {
pub fn get_language_grammar(language: Language) -> Result<tree_sitter::Language> {
match language {
Language::Rust => Ok(tree_sitter_rust::LANGUAGE.into()),
Language::Python => Ok(tree_sitter_python::LANGUAGE.into()),
Language::TypeScript => Ok(tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into()),
Language::JavaScript => Ok(tree_sitter_typescript::LANGUAGE_TSX.into()),
Language::Go => Ok(tree_sitter_go::LANGUAGE.into()),
Language::Java => Ok(tree_sitter_java::LANGUAGE.into()),
Language::C => Ok(tree_sitter_c::LANGUAGE.into()),
Language::Cpp => Ok(tree_sitter_cpp::LANGUAGE.into()),
Language::CSharp => Ok(tree_sitter_c_sharp::LANGUAGE.into()),
Language::PHP => Ok(tree_sitter_php::LANGUAGE_PHP.into()),
Language::Ruby => Ok(tree_sitter_ruby::LANGUAGE.into()),
Language::Kotlin => Ok(tree_sitter_kotlin_ng::LANGUAGE.into()),
Language::Zig => Ok(tree_sitter_zig::LANGUAGE.into()),
Language::Swift => Err(anyhow!(
"Swift support temporarily disabled (parser queries out of date with tree-sitter-swift 0.7.x grammar)"
)),
Language::Vue => Err(anyhow!(
"Vue uses line-based parsing, not tree-sitter (tree-sitter-vue incompatible with tree-sitter 0.24+)"
)),
Language::Svelte => Err(anyhow!(
"Svelte uses line-based parsing, not tree-sitter (tree-sitter-svelte incompatible with tree-sitter 0.24+)"
)),
Language::Text | Language::Lock | Language::Generated => Err(anyhow!(
"The text tier (docs, config, templates, lock and generated files) is \
trigram-indexed only and has no grammar. Use full-text or regex \
search on these files, not --symbols or --ast."
)),
Language::Unknown => Err(anyhow!("Unknown language")),
}
}
pub fn get_keywords(language: Language) -> &'static [&'static str] {
match language {
Language::Rust => &[
"fn", "struct", "enum", "trait", "impl", "mod", "const", "static", "type", "macro",
],
Language::PHP => &["class", "function", "trait", "interface", "enum"],
Language::Python => &["class", "def", "async"],
Language::TypeScript | Language::JavaScript => &[
"class",
"function",
"interface",
"type",
"enum",
"const",
"let",
"var",
],
Language::Go => &["func", "struct", "interface", "type", "const", "var"],
Language::Java => &["class", "interface", "enum", "@interface"],
Language::C => &["struct", "enum", "union", "typedef"],
Language::Cpp => &[
"class",
"struct",
"enum",
"union",
"typedef",
"namespace",
"template",
],
Language::CSharp => &[
"class",
"struct",
"interface",
"enum",
"delegate",
"record",
"namespace",
],
Language::Ruby => &["class", "module", "def"],
Language::Kotlin => &["class", "fun", "interface", "object", "enum", "annotation"],
Language::Zig => &["fn", "struct", "enum", "const", "var", "type"],
Language::Swift => &["class", "struct", "enum", "protocol", "func", "var", "let"],
Language::Vue | Language::Svelte => &["function", "const", "let", "var"],
Language::Text | Language::Lock | Language::Generated => &[],
Language::Unknown => &[],
}
}
pub fn get_all_keywords() -> &'static [&'static str] {
&[
"fn",
"function",
"def",
"func",
"class",
"struct",
"enum",
"interface",
"trait",
"type",
"record",
"mod",
"module",
"namespace",
"const",
"static",
"let",
"var",
"impl",
"async",
"object",
"annotation",
"protocol",
"union",
"typedef",
"delegate",
"template",
"@interface",
]
}
pub fn has_symbol_parser(language: Language) -> bool {
!matches!(
language,
Language::Swift
| Language::Text
| Language::Lock
| Language::Generated
| Language::Unknown
)
}
pub fn parse(path: &str, source: &str, language: Language) -> Result<Vec<SearchResult>> {
if is_minified(source) {
log::info!(
"Skipping symbol extraction for {} — looks minified ({} bytes over {} lines). \
The file is still searchable with full-text and regex queries.",
path,
source.len(),
count_lines(source)
);
return Ok(Vec::new());
}
if language == Language::TypeScript && is_qt_linguist_ts(source) {
log::debug!(
"Skipping symbol extraction for {} — a Qt Linguist translation file (XML), \
not TypeScript. The file is still searchable.",
path
);
return Ok(Vec::new());
}
match language {
Language::Rust => rust::parse(path, source),
Language::TypeScript => typescript::parse(path, source, language),
Language::JavaScript => typescript::parse(path, source, language),
Language::Vue => vue::parse(path, source),
Language::Svelte => svelte::parse(path, source),
Language::Python => python::parse(path, source),
Language::Go => go::parse(path, source),
Language::Java => java::parse(path, source),
Language::PHP => php::parse(path, source),
Language::C => c::parse(path, source),
Language::Cpp => cpp::parse(path, source),
Language::CSharp => csharp::parse(path, source),
Language::Ruby => ruby::parse(path, source),
Language::Kotlin => kotlin::parse(path, source),
Language::Swift => {
log::warn!(
"Swift support temporarily disabled (parser queries out of date with tree-sitter-swift 0.7.x grammar): {}",
path
);
Ok(vec![])
}
Language::Zig => zig::parse(path, source),
Language::Text | Language::Lock | Language::Generated => {
log::debug!("No symbol extraction for text-tier file: {}", path);
Ok(vec![])
}
Language::Unknown => {
log::warn!("Unknown language for file: {}", path);
Ok(vec![])
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parser_factory() {
let _factory = ParserFactory;
}
}
pub const MAX_BYTES_PER_LINE: usize = 1024;
pub const MINIFIED_SIZE_FLOOR: usize = 16 * 1024;
fn count_lines(source: &str) -> usize {
memchr::memchr_iter(b'\n', source.as_bytes()).count()
}
pub fn is_minified(source: &str) -> bool {
if source.len() < MINIFIED_SIZE_FLOOR {
return false;
}
let enough = source.len() / MAX_BYTES_PER_LINE;
let mut seen = 0usize;
for _ in memchr::memchr_iter(b'\n', source.as_bytes()) {
seen += 1;
if seen > enough {
return false;
}
}
true
}
pub fn is_qt_linguist_ts(source: &str) -> bool {
let head = source.trim_start_matches('\u{feff}').trim_start();
head.starts_with("<?xml") || head.starts_with("<!DOCTYPE")
}
#[cfg(test)]
mod qt_linguist_tests {
use super::*;
const QT_TS: &str = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<!DOCTYPE TS>\n\
<TS version=\"2.1\" language=\"de_DE\">\n<context>\n <name>MainWindow</name>\n\
<message>\n <source>Open</source>\n <translation>Öffnen</translation>\n\
</message>\n</context>\n</TS>\n";
#[test]
fn qt_linguist_files_are_recognised() {
assert!(is_qt_linguist_ts(QT_TS));
assert!(is_qt_linguist_ts(&format!("\u{feff}{QT_TS}")));
assert!(is_qt_linguist_ts(
"\n <!DOCTYPE TS>\n<TS version=\"2.0\">\n</TS>\n"
));
}
#[test]
fn typescript_is_not_mistaken_for_xml() {
for src in [
"export const a = 1;\n",
"// <?xml version=\"1.0\"?>\nexport const a = 1;\n",
"const xml = `<?xml version=\"1.0\"?>`;\n",
"",
] {
assert!(!is_qt_linguist_ts(src), "{src}");
}
}
#[test]
fn a_qt_linguist_file_yields_no_symbols() {
let out = ParserFactory::parse("app_de.ts", QT_TS, Language::TypeScript).unwrap();
assert!(out.is_empty(), "got {} symbols", out.len());
}
}
#[cfg(test)]
mod minified_tests {
use super::*;
#[test]
fn real_source_is_never_minified() {
let src = "pub fn some_function_name(a: u32) -> u32 {\n".repeat(2000);
assert!(!is_minified(&src));
}
#[test]
fn a_one_line_bundle_is_minified() {
assert!(is_minified(&"function f(a,b){return a+b}".repeat(60_000)));
}
#[test]
fn a_small_file_is_never_minified_however_long_its_line() {
assert!(!is_minified(&"x".repeat(MINIFIED_SIZE_FLOOR - 1)));
}
#[test]
fn one_long_line_among_normal_ones_is_not_minified() {
let src = format!("{}\n{}", "z".repeat(12_282), "const a = 1;\n".repeat(2000));
assert!(
!is_minified(&src),
"generated protobuf must keep its symbols"
);
}
#[test]
fn cjk_source_is_not_minified() {
let line = format!("// {}\n", "日".repeat(200));
assert!(!is_minified(&line.repeat(200)));
}
#[test]
fn a_minified_file_yields_no_symbols_but_does_not_error() {
let src = "function f(a,b){return a+b}".repeat(60_000);
let out = ParserFactory::parse("bundle.js", &src, Language::JavaScript).unwrap();
assert!(out.is_empty(), "got {} symbols", out.len());
}
#[test]
fn a_normal_file_still_yields_symbols() {
let src = "export function realOne(a: number) {\n return a;\n}\n".repeat(50);
let out = ParserFactory::parse("real.ts", &src, Language::TypeScript).unwrap();
assert!(!out.is_empty(), "normal source must still parse");
}
}
pub type CachedQuery = std::sync::OnceLock<std::result::Result<tree_sitter::Query, String>>;
pub fn cached_query<'a>(
cell: &'a CachedQuery,
language: impl Into<tree_sitter::Language>,
source: &str,
) -> Result<&'a tree_sitter::Query> {
match cell.get_or_init(|| {
tree_sitter::Query::new(&language.into(), source).map_err(|e| e.to_string())
}) {
Ok(query) => Ok(query),
Err(e) => Err(anyhow!("{e}")),
}
}
pub struct KeyedQueries {
cells: std::sync::Mutex<Vec<(tree_sitter::Language, &'static tree_sitter::Query)>>,
}
impl KeyedQueries {
pub const fn new() -> Self {
Self {
cells: std::sync::Mutex::new(Vec::new()),
}
}
pub fn get(
&'static self,
language: &tree_sitter::Language,
source: &str,
) -> Result<&'static tree_sitter::Query> {
let mut cells = self.cells.lock().unwrap_or_else(|e| e.into_inner());
if let Some((_, query)) = cells.iter().find(|(l, _)| l == language) {
return Ok(query);
}
let query = tree_sitter::Query::new(language, source).map_err(|e| anyhow!("{e}"))?;
let query: &'static tree_sitter::Query = Box::leak(Box::new(query));
cells.push((language.clone(), query));
Ok(query)
}
}
impl Default for KeyedQueries {
fn default() -> Self {
Self::new()
}
}
pub struct CaptureRec<'t> {
pub index: u32,
pub node: tree_sitter::Node<'t>,
}
pub struct MatchRec<'t> {
pub pattern_index: usize,
pub captures: Vec<CaptureRec<'t>>,
}
pub struct MatchTable<'t> {
query: &'static tree_sitter::Query,
subs: Vec<Vec<MatchRec<'t>>>,
}
impl<'t> MatchTable<'t> {
pub fn query(&self) -> &'static tree_sitter::Query {
self.query
}
pub fn sub(&self, k: usize) -> &[MatchRec<'t>] {
self.subs.get(k).map(Vec::as_slice).unwrap_or(&[])
}
}
struct CombinedQuery {
query: &'static tree_sitter::Query,
sub_of: Vec<usize>,
}
pub struct LanguageQueries {
sources: &'static [&'static str],
compiled: std::sync::Mutex<Vec<(tree_sitter::Language, &'static CombinedQuery)>>,
}
impl LanguageQueries {
pub const fn new(sources: &'static [&'static str]) -> Self {
Self {
sources,
compiled: std::sync::Mutex::new(Vec::new()),
}
}
fn compiled_for(
&'static self,
language: &tree_sitter::Language,
) -> Result<&'static CombinedQuery> {
let mut cells = self.compiled.lock().unwrap_or_else(|e| e.into_inner());
if let Some((_, c)) = cells.iter().find(|(l, _)| l == language) {
return Ok(c);
}
let mut sub_of = Vec::new();
let mut combined = String::new();
for (k, src) in self.sources.iter().enumerate() {
let alone =
tree_sitter::Query::new(language, src).map_err(|e| anyhow!("query {k}: {e}"))?;
sub_of.extend(std::iter::repeat_n(k, alone.pattern_count()));
combined.push_str(src);
combined.push('\n');
}
let query = tree_sitter::Query::new(language, &combined)
.map_err(|e| anyhow!("combined query: {e}"))?;
if query.pattern_count() != sub_of.len() {
anyhow::bail!(
"combined query has {} patterns, sub-queries have {}",
query.pattern_count(),
sub_of.len()
);
}
let c: &'static CombinedQuery = Box::leak(Box::new(CombinedQuery {
query: Box::leak(Box::new(query)),
sub_of,
}));
cells.push((language.clone(), c));
Ok(c)
}
pub fn run<'t>(
&'static self,
language: &tree_sitter::Language,
root: &tree_sitter::Node<'t>,
source: &str,
) -> Result<MatchTable<'t>> {
use streaming_iterator::StreamingIterator;
let c = self.compiled_for(language)?;
let mut subs: Vec<Vec<MatchRec<'t>>> =
(0..self.sources.len()).map(|_| Vec::new()).collect();
let mut cursor = tree_sitter::QueryCursor::new();
let mut matches = cursor.matches(c.query, *root, source.as_bytes());
while let Some(m) = matches.next() {
let pattern_index = m.pattern_index;
let captures = m
.captures
.iter()
.map(|cap| CaptureRec {
index: cap.index,
node: cap.node,
})
.collect();
subs[c.sub_of[pattern_index]].push(MatchRec {
pattern_index,
captures,
});
}
Ok(MatchTable {
query: c.query,
subs,
})
}
}