pub mod config_extractors;
pub mod languages;
pub mod xref;
use std::collections::HashMap;
use std::panic;
use tree_sitter::Parser;
use tree_sitter::Node;
use crate::domain::{
FileClassification, FileOutcome, FileProcessingResult, LanguageId, ParseDiagnostic,
ReferenceRecord, SymbolRecord,
};
use crate::hash::digest_hex;
type ParseSourceOutput = (
Vec<SymbolRecord>,
bool,
Option<ParseDiagnostic>,
Vec<ReferenceRecord>,
HashMap<String, String>,
);
pub fn process_file(
relative_path: &str,
bytes: &[u8],
language: LanguageId,
) -> FileProcessingResult {
process_file_with_classification(
relative_path,
bytes,
language,
FileClassification::for_code_path(relative_path),
)
}
pub fn process_file_with_classification(
relative_path: &str,
bytes: &[u8],
language: LanguageId,
classification: FileClassification,
) -> FileProcessingResult {
let byte_len = bytes.len() as u64;
let content_hash = digest_hex(bytes);
if config_extractors::is_config_language(&language) {
let result = config_extractors::extractor_for(&language).map(|e| e.extract(bytes));
let (symbols, outcome, parse_diagnostic) = match result {
Some(r) => {
let (outcome, parse_diagnostic) = match r.outcome {
config_extractors::ExtractionOutcome::Ok => (FileOutcome::Processed, None),
config_extractors::ExtractionOutcome::Partial(diagnostic) => (
FileOutcome::PartialParse {
warning: diagnostic.summary(),
},
Some(diagnostic),
),
config_extractors::ExtractionOutcome::Failed(diagnostic) => (
FileOutcome::Failed {
error: diagnostic.summary(),
},
Some(diagnostic),
),
};
(r.symbols, outcome, parse_diagnostic)
}
None => (vec![], FileOutcome::Processed, None),
};
return FileProcessingResult {
relative_path: relative_path.to_string(),
language,
classification,
outcome,
parse_diagnostic,
symbols,
byte_len,
content_hash,
references: vec![],
alias_map: HashMap::new(),
};
}
let source = String::from_utf8_lossy(bytes);
let parse_result = panic::catch_unwind(|| parse_source(&source, &language));
match parse_result {
Ok(Ok((symbols, has_error, diagnostic, references, alias_map))) => {
let outcome = if has_error {
let warning = diagnostic
.as_ref()
.map(|d| d.summary())
.unwrap_or_else(|| {
"tree-sitter reported syntax errors in the parse tree".to_string()
});
FileOutcome::PartialParse { warning }
} else {
FileOutcome::Processed
};
FileProcessingResult {
relative_path: relative_path.to_string(),
language,
classification,
outcome,
parse_diagnostic: diagnostic,
symbols,
byte_len,
content_hash,
references,
alias_map,
}
}
Ok(Err(err)) => FileProcessingResult {
relative_path: relative_path.to_string(),
language,
classification,
outcome: FileOutcome::Failed {
error: err.to_string(),
},
parse_diagnostic: None,
symbols: vec![],
byte_len,
content_hash,
references: vec![],
alias_map: HashMap::new(),
},
Err(_panic) => FileProcessingResult {
relative_path: relative_path.to_string(),
language,
classification,
outcome: FileOutcome::Failed {
error: "tree-sitter parser panicked during parsing".to_string(),
},
parse_diagnostic: None,
symbols: vec![],
byte_len,
content_hash,
references: vec![],
alias_map: HashMap::new(),
},
}
}
fn collect_first_error_node(root: &Node, source: &str) -> Option<(String, u32, u32, (u32, u32))> {
let mut cursor = root.walk();
let mut stack = vec![*root];
while let Some(node) = stack.pop() {
if node.is_error() || node.is_missing() {
let start = node.start_position();
let snippet_start = node.start_byte();
let snippet_end = node.end_byte().min(snippet_start + 40);
let snippet = &source[snippet_start..snippet_end];
let kind = if node.is_missing() { "missing" } else { "error" };
let message = format!(
"syntax {kind} near `{}`",
snippet.replace('\n', "\\n")
);
return Some((
message,
start.row as u32 + 1, start.column as u32 + 1, (node.start_byte() as u32, node.end_byte() as u32),
));
}
cursor.reset(node);
if cursor.goto_first_child() {
let mut children = vec![cursor.node()];
while cursor.goto_next_sibling() {
children.push(cursor.node());
}
stack.extend(children.into_iter().rev());
}
}
None
}
fn parse_source(source: &str, language: &LanguageId) -> Result<ParseSourceOutput, String> {
let mut parser = Parser::new();
let ts_language = match language {
LanguageId::Rust => tree_sitter_rust::LANGUAGE.into(),
LanguageId::Python => tree_sitter_python::LANGUAGE.into(),
LanguageId::JavaScript => tree_sitter_javascript::LANGUAGE.into(),
LanguageId::TypeScript => tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(),
LanguageId::Go => tree_sitter_go::LANGUAGE.into(),
LanguageId::Java => tree_sitter_java::LANGUAGE.into(),
LanguageId::C => tree_sitter_c::LANGUAGE.into(),
LanguageId::Cpp => tree_sitter_cpp::LANGUAGE.into(),
LanguageId::CSharp => tree_sitter_c_sharp::LANGUAGE.into(),
LanguageId::Ruby => tree_sitter_ruby::LANGUAGE.into(),
LanguageId::Php => tree_sitter_php::LANGUAGE_PHP.into(),
LanguageId::Swift => tree_sitter_swift::LANGUAGE.into(),
LanguageId::Perl => tree_sitter_perl::LANGUAGE.into(),
LanguageId::Kotlin => tree_sitter_kotlin_sg::LANGUAGE.into(),
LanguageId::Dart => tree_sitter_dart::language(),
LanguageId::Elixir => tree_sitter_elixir::LANGUAGE.into(),
LanguageId::Json
| LanguageId::Toml
| LanguageId::Yaml
| LanguageId::Markdown
| LanguageId::Env => unreachable!("config types are handled before parse_source"),
LanguageId::Html => tree_sitter_html::LANGUAGE.into(),
LanguageId::Css => tree_sitter_css::LANGUAGE.into(),
LanguageId::Scss => tree_sitter_scss::language(),
};
parser
.set_language(&ts_language)
.map_err(|e| format!("failed to set language: {e}"))?;
let tree = parser
.parse(source, None)
.ok_or_else(|| "tree-sitter parse returned None".to_string())?;
let root = tree.root_node();
let has_error = root.has_error();
let symbols = languages::extract_symbols(&root, source, language);
let (references, alias_map) = xref::extract_references(&root, source, language);
let diagnostic = if has_error {
collect_first_error_node(&root, source).map(|(message, line, column, span)| {
ParseDiagnostic {
parser: "tree-sitter".to_string(),
message,
line: Some(line),
column: Some(column),
byte_span: Some(span),
fallback_used: false,
}
})
} else {
None
};
Ok((symbols, has_error, diagnostic, references, alias_map))
}
pub fn extract_symbols_for_diff(source: &str, path: &str) -> Option<Vec<(String, String)>> {
let ext = path.rsplit('.').next().unwrap_or("");
let language = LanguageId::from_extension(ext)?;
if config_extractors::is_config_language(&language) {
return None; }
let result = panic::catch_unwind(|| parse_source(source, &language));
let (symbols, ..) = match result {
Ok(Ok(output)) => output,
_ => return None,
};
let pairs: Vec<(String, String)> = symbols
.iter()
.map(|sym| {
let (start, end) = sym.byte_range;
let body = &source[start as usize..end as usize];
(sym.name.clone(), crate::hash::digest_hex(body.as_bytes()))
})
.collect();
Some(pairs)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::domain::{FileOutcome, LanguageId, SymbolKind};
#[test]
fn test_process_file_rust_extracts_function() {
let source = b"fn hello() { }";
let result = process_file("test.rs", source, LanguageId::Rust);
assert_eq!(result.outcome, FileOutcome::Processed);
assert!(!result.symbols.is_empty());
assert_eq!(result.symbols[0].name, "hello");
assert_eq!(result.symbols[0].kind, SymbolKind::Function);
}
#[test]
fn test_process_file_python_extracts_function() {
let source = b"def greet():\n pass";
let result = process_file("test.py", source, LanguageId::Python);
assert_eq!(result.outcome, FileOutcome::Processed);
assert!(!result.symbols.is_empty());
assert_eq!(result.symbols[0].name, "greet");
assert_eq!(result.symbols[0].kind, SymbolKind::Function);
}
#[test]
fn test_process_file_javascript_extracts_function() {
let source = b"function doStuff() { }";
let result = process_file("test.js", source, LanguageId::JavaScript);
assert_eq!(result.outcome, FileOutcome::Processed);
assert!(!result.symbols.is_empty());
assert_eq!(result.symbols[0].name, "doStuff");
assert_eq!(result.symbols[0].kind, SymbolKind::Function);
}
#[test]
fn test_process_file_typescript_extracts_interface() {
let source = b"interface Greeter { greet(): void; }";
let result = process_file("test.ts", source, LanguageId::TypeScript);
assert_eq!(result.outcome, FileOutcome::Processed);
let interface = result
.symbols
.iter()
.find(|s| s.kind == SymbolKind::Interface);
assert!(interface.is_some());
assert_eq!(interface.unwrap().name, "Greeter");
}
#[test]
fn test_process_file_go_extracts_function() {
let source = b"package main\nfunc main() { }";
let result = process_file("test.go", source, LanguageId::Go);
assert_eq!(result.outcome, FileOutcome::Processed);
assert!(!result.symbols.is_empty());
assert_eq!(result.symbols[0].name, "main");
assert_eq!(result.symbols[0].kind, SymbolKind::Function);
}
#[test]
fn test_process_file_partial_parse() {
let source = b"fn broken( { }";
let result = process_file("bad.rs", source, LanguageId::Rust);
assert!(matches!(result.outcome, FileOutcome::PartialParse { .. }));
}
#[test]
fn test_process_file_partial_parse_has_diagnostic() {
let source = b"fn broken( { }";
let result = process_file("bad.rs", source, LanguageId::Rust);
assert!(matches!(result.outcome, FileOutcome::PartialParse { .. }));
let diag = result.parse_diagnostic.expect("should have a diagnostic for partial parse");
assert_eq!(diag.parser, "tree-sitter");
assert!(diag.line.is_some(), "diagnostic should have a line number");
assert!(diag.column.is_some(), "diagnostic should have a column number");
assert!(diag.byte_span.is_some(), "diagnostic should have a byte span");
assert!(diag.message.contains("syntax"), "message should describe the error");
}
#[test]
fn test_process_file_computes_content_hash() {
let source = b"fn foo() {}";
let result = process_file("hash_test.rs", source, LanguageId::Rust);
assert!(!result.content_hash.is_empty());
assert_eq!(result.content_hash, digest_hex(source));
}
#[test]
fn test_process_file_byte_len() {
let source = b"fn bar() {}";
let result = process_file("len.rs", source, LanguageId::Rust);
assert_eq!(result.byte_len, source.len() as u64);
}
#[test]
fn test_process_file_preserves_relative_path() {
let result = process_file("src/lib.rs", b"fn x() {}", LanguageId::Rust);
assert_eq!(result.relative_path, "src/lib.rs");
}
#[test]
fn test_process_file_never_panics_on_adversarial_input() {
let cases: &[(&[u8], &str, LanguageId)] = &[
(b"\xff\xfe\x00\x01", "binary.rs", LanguageId::Rust),
(b"", "empty.py", LanguageId::Python),
(&[0u8; 10000], "zeros.js", LanguageId::JavaScript),
(b"\n\n\n\n\n", "newlines.ts", LanguageId::TypeScript),
("\u{200b}\u{200b}".as_bytes(), "zwsp.go", LanguageId::Go),
(
b"\0\0\0fn main() {}\0\0",
"null_padded.rs",
LanguageId::Rust,
),
];
for &(source, path, ref lang) in cases {
let result = process_file(path, source, lang.clone());
assert_eq!(result.relative_path, path);
assert_eq!(result.byte_len, source.len() as u64);
assert!(!result.content_hash.is_empty());
}
}
#[test]
fn test_process_file_ruby_extracts_method() {
let source = b"def hello\n puts 'hi'\nend";
let result = process_file("app.rb", source, LanguageId::Ruby);
assert_eq!(result.outcome, FileOutcome::Processed);
assert!(
!result.symbols.is_empty(),
"should have symbols for Ruby source"
);
}
}