pub mod chunker;
pub mod extractor;
pub mod languages;
pub mod references;
pub mod signatures;
pub mod sphinx;
pub mod type_relations;
use anyhow::{Context, Result};
use tree_sitter::Parser;
use crate::config::IndexingConfig;
use crate::types::{Chunk, Language};
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct ParseDiagnostics {
pub tree_has_error: bool,
pub used_tier0_fallback: bool,
}
pub fn parse_file_with_diagnostics(
source: &str,
file_path: &str,
language: Language,
config: &IndexingConfig,
) -> Result<(Vec<Chunk>, Vec<references::RawReference>, ParseDiagnostics)> {
if language == Language::Markdown {
let chunks = chunker::markdown_section_chunks(source, file_path);
return Ok((
chunker::split_oversized_chunks(chunks, config.max_chunk_bytes),
Vec::new(),
ParseDiagnostics::default(),
));
}
if language == Language::Rst {
let (chunks, diagnostics) = parse_rst_section_chunks(source, file_path)?;
return Ok((
chunker::split_oversized_chunks(chunks, config.max_chunk_bytes),
Vec::new(),
diagnostics,
));
}
if language.prefers_file_chunking() {
let chunks = chunker::whole_file_chunk(source, file_path, language);
return Ok((
chunker::split_oversized_chunks(chunks, config.max_chunk_bytes),
Vec::new(),
ParseDiagnostics::default(),
));
}
if uses_indexing_tier0_fallback(language) {
let chunks = chunker::tier0_line_chunks(source, file_path, language);
return Ok((
chunker::split_oversized_chunks(chunks, config.max_chunk_bytes),
Vec::new(),
ParseDiagnostics {
used_tier0_fallback: true,
..ParseDiagnostics::default()
},
));
}
let grammar = match languages::tree_sitter_grammar_for_path(language, file_path) {
Some(g) => g,
None => {
let chunks = chunker::tier0_line_chunks(source, file_path, language);
return Ok((
chunker::split_oversized_chunks(chunks, config.max_chunk_bytes),
Vec::new(),
ParseDiagnostics {
used_tier0_fallback: true,
..ParseDiagnostics::default()
},
));
}
};
let mut parser = Parser::new();
parser
.set_language(&grammar)
.context("failed to load tree-sitter grammar")?;
let tree = parser
.parse(source, None)
.context("tree-sitter parsing returned None")?;
let tree_has_error = tree.root_node().has_error();
let symbols = extractor::extract_symbols(&tree, source.as_bytes(), language);
let refs =
references::extract_references_with_symbols(&tree, source.as_bytes(), language, &symbols);
let chunks = chunker::chunks_from_symbols(&symbols, source, file_path, language, config);
let used_tier0_fallback = chunks.is_empty() && !source.trim().is_empty();
let chunks = if used_tier0_fallback {
chunker::tier0_line_chunks(source, file_path, language)
} else {
chunks
};
Ok((
chunker::split_oversized_chunks(chunks, config.max_chunk_bytes),
refs,
ParseDiagnostics {
tree_has_error,
used_tier0_fallback,
},
))
}
fn uses_indexing_tier0_fallback(language: Language) -> bool {
matches!(language, Language::Haskell)
}
pub fn parse_and_chunk(
source: &str,
file_path: &str,
language: Language,
config: &IndexingConfig,
) -> Result<Vec<Chunk>> {
let (chunks, _refs, _diagnostics) =
parse_file_with_diagnostics(source, file_path, language, config)?;
Ok(chunks)
}
fn parse_rst_section_chunks(
source: &str,
file_path: &str,
) -> Result<(Vec<Chunk>, ParseDiagnostics)> {
let grammar = languages::tree_sitter_grammar(Language::Rst)
.context("missing tree-sitter grammar for reStructuredText")?;
let mut parser = Parser::new();
parser
.set_language(&grammar)
.context("failed to load reStructuredText grammar")?;
let tree = parser
.parse(source, None)
.context("tree-sitter parsing returned None")?;
let tree_has_error = tree.root_node().has_error();
let headings = extractor::extract_rst_section_titles(&tree, source.as_bytes());
if headings.is_empty() {
return Ok((
chunker::tier0_line_chunks(source, file_path, Language::Rst),
ParseDiagnostics {
tree_has_error,
used_tier0_fallback: true,
},
));
}
Ok((
chunker::rst_section_chunks(source, file_path, &headings),
ParseDiagnostics {
tree_has_error,
used_tier0_fallback: false,
},
))
}
pub fn parse_and_extract_references(
source: &str,
language: Language,
) -> Vec<references::RawReference> {
let grammar = match languages::tree_sitter_grammar(language) {
Some(g) => g,
None => return Vec::new(),
};
let mut parser = Parser::new();
if parser.set_language(&grammar).is_err() {
return Vec::new();
}
let tree = match parser.parse(source, None) {
Some(t) => t,
None => return Vec::new(),
};
references::extract_references(&tree, source.as_bytes(), language)
}
#[cfg(test)]
pub(crate) mod test_support;
#[cfg(test)]
mod tests;
#[cfg(test)]
mod metadata_tests;