pub mod ast_grep;
pub mod config_extractors;
#[cfg(test)]
mod inline_tests;
pub mod languages;
#[cfg(feature = "cbm-spike")]
pub mod resolver;
pub mod xref;
use std::collections::HashMap;
use std::panic;
use tree_sitter::Parser;
use tree_sitter::Node;
use crate::domain::{
FileClassification, FileOutcome, FileProcessingResult, LanguageId, ParseDiagnostic,
ReferenceRecord, SymbolRecord,
};
use crate::hash::digest_hex;
type ParseSourceOutput = (
Vec<SymbolRecord>,
bool,
Option<ParseDiagnostic>,
Vec<ReferenceRecord>,
HashMap<String, String>,
);
pub fn process_file(
relative_path: &str,
bytes: &[u8],
language: LanguageId,
) -> FileProcessingResult {
process_file_with_classification(
relative_path,
bytes,
language,
FileClassification::for_code_path(relative_path),
)
}
pub fn process_file_with_classification(
relative_path: &str,
bytes: &[u8],
language: LanguageId,
classification: FileClassification,
) -> FileProcessingResult {
let byte_len = bytes.len() as u64;
let content_hash = digest_hex(bytes);
if config_extractors::is_config_language(&language) {
let result = config_extractors::extractor_for(&language).map(|e| e.extract(bytes));
let (symbols, outcome, parse_diagnostic) = match result {
Some(r) => {
let (outcome, parse_diagnostic) = match r.outcome {
config_extractors::ExtractionOutcome::Ok => (FileOutcome::Processed, None),
config_extractors::ExtractionOutcome::Partial(diagnostic) => (
FileOutcome::PartialParse {
warning: diagnostic.summary(),
},
Some(diagnostic),
),
config_extractors::ExtractionOutcome::Failed(diagnostic) => (
FileOutcome::Failed {
error: diagnostic.summary(),
},
Some(diagnostic),
),
};
(r.symbols, outcome, parse_diagnostic)
}
None => (vec![], FileOutcome::Processed, None),
};
return FileProcessingResult {
relative_path: relative_path.to_string(),
language,
classification,
outcome,
parse_diagnostic,
symbols,
byte_len,
content_hash,
references: vec![],
alias_map: HashMap::new(),
};
}
let source = String::from_utf8_lossy(bytes);
let is_tsx = LanguageId::is_tsx_path(relative_path);
let (language, parse_result) =
if matches!(language, LanguageId::C) && LanguageId::is_c_header_path(relative_path) {
resolve_c_header_parse(&source)
} else {
let parse_result = panic::catch_unwind(|| parse_source(&source, &language, is_tsx));
(language, parse_result)
};
match parse_result {
Ok(Ok((symbols, has_error, diagnostic, references, alias_map))) => {
let outcome = if has_error {
let warning = diagnostic.as_ref().map(|d| d.summary()).unwrap_or_else(|| {
"tree-sitter reported syntax errors in the parse tree".to_string()
});
FileOutcome::PartialParse { warning }
} else {
FileOutcome::Processed
};
FileProcessingResult {
relative_path: relative_path.to_string(),
language,
classification,
outcome,
parse_diagnostic: diagnostic,
symbols,
byte_len,
content_hash,
references,
alias_map,
}
}
Ok(Err(err)) => FileProcessingResult {
relative_path: relative_path.to_string(),
language,
classification,
outcome: FileOutcome::Failed {
error: err.to_string(),
},
parse_diagnostic: None,
symbols: vec![],
byte_len,
content_hash,
references: vec![],
alias_map: HashMap::new(),
},
Err(_panic) => FileProcessingResult {
relative_path: relative_path.to_string(),
language,
classification,
outcome: FileOutcome::Failed {
error: "tree-sitter parser panicked during parsing".to_string(),
},
parse_diagnostic: None,
symbols: vec![],
byte_len,
content_hash,
references: vec![],
alias_map: HashMap::new(),
},
}
}
fn collect_deepest_error_node(root: &Node, source: &str) -> Option<(String, u32, u32, (u32, u32))> {
let mut cursor = root.walk();
let mut stack = vec![(*root, 0usize)];
let mut best: Option<(Node, usize)> = None;
while let Some((node, depth)) = stack.pop() {
if node.is_error() || node.is_missing() {
best = match best {
Some((current, current_depth))
if !error_candidate_is_better(node, depth, current, current_depth) =>
{
Some((current, current_depth))
}
_ => Some((node, depth)),
};
}
cursor.reset(node);
if cursor.goto_first_child() {
let mut children = vec![cursor.node()];
while cursor.goto_next_sibling() {
children.push(cursor.node());
}
stack.extend(children.into_iter().rev().map(|child| (child, depth + 1)));
}
}
best.map(|(node, _depth)| {
let start = node.start_position();
let snippet_start = node.start_byte().min(source.len());
let snippet_limit = snippet_start + 40;
let span_end = node.end_byte().min(source.len());
let mut snippet_end = if span_end > snippet_start {
span_end.min(snippet_limit)
} else {
snippet_limit.min(source.len())
};
while snippet_end > snippet_start && !source.is_char_boundary(snippet_end) {
snippet_end -= 1;
}
let snippet = &source[snippet_start..snippet_end];
let kind = if node.is_missing() {
format!("missing {}", node.kind())
} else {
"error".to_string()
};
let message = format!("syntax {kind} near `{}`", snippet.replace('\n', "\\n"));
(
message,
start.row as u32 + 1, start.column as u32 + 1, (node.start_byte() as u32, node.end_byte() as u32),
)
})
}
fn augment_message_with_nul_hint(message: String, source: &[u8]) -> String {
let scan = crate::domain::index::scan_nul_bytes(source);
if !scan.has_nul() {
return message;
}
let offsets = scan
.first_offsets
.iter()
.map(|o| o.to_string())
.collect::<Vec<_>>()
.join(", ");
let offset_suffix = if scan.count > scan.first_offsets.len() {
format!("{offsets}, ...")
} else {
offsets
};
let plural = if scan.count == 1 { "" } else { "s" };
format!(
"{message} (file contains {} NUL byte{plural} at offset(s) {offset_suffix}; \
tree-sitter cannot parse NUL in source)",
scan.count
)
}
fn error_candidate_is_better(
candidate: Node,
candidate_depth: usize,
current: Node,
current_depth: usize,
) -> bool {
if candidate_depth != current_depth {
return candidate_depth > current_depth;
}
let candidate_len = candidate.end_byte().saturating_sub(candidate.start_byte());
let current_len = current.end_byte().saturating_sub(current.start_byte());
if candidate_len != current_len {
return candidate_len < current_len;
}
if candidate.start_byte() != current.start_byte() {
return candidate.start_byte() < current.start_byte();
}
candidate.is_missing() && !current.is_missing()
}
pub(crate) fn parse_source(
source: &str,
language: &LanguageId,
is_tsx: bool,
) -> Result<ParseSourceOutput, String> {
let mut parser = Parser::new();
let ts_language = match language {
LanguageId::Rust => tree_sitter_rust::LANGUAGE.into(),
LanguageId::Python => tree_sitter_python::LANGUAGE.into(),
LanguageId::JavaScript => tree_sitter_javascript::LANGUAGE.into(),
LanguageId::TypeScript if is_tsx => tree_sitter_typescript::LANGUAGE_TSX.into(),
LanguageId::TypeScript => tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(),
LanguageId::Go => tree_sitter_go::LANGUAGE.into(),
LanguageId::Java => tree_sitter_java::LANGUAGE.into(),
LanguageId::C => tree_sitter_c::LANGUAGE.into(),
LanguageId::Cpp => tree_sitter_cpp::LANGUAGE.into(),
LanguageId::CSharp => tree_sitter_c_sharp::LANGUAGE.into(),
LanguageId::Ruby => tree_sitter_ruby::LANGUAGE.into(),
LanguageId::Php => tree_sitter_php::LANGUAGE_PHP.into(),
LanguageId::Swift => tree_sitter_swift::LANGUAGE.into(),
LanguageId::Perl => tree_sitter_perl::LANGUAGE.into(),
LanguageId::Kotlin => tree_sitter_kotlin_sg::LANGUAGE.into(),
LanguageId::Dart => tree_sitter_dart::LANGUAGE.into(),
LanguageId::Elixir => tree_sitter_elixir::LANGUAGE.into(),
LanguageId::Json
| LanguageId::Toml
| LanguageId::Yaml
| LanguageId::Markdown
| LanguageId::Env => unreachable!("config types are handled before parse_source"),
LanguageId::Html => tree_sitter_html::LANGUAGE.into(),
LanguageId::Css => tree_sitter_css::LANGUAGE.into(),
LanguageId::Scss => tree_sitter_scss::language(),
};
parser
.set_language(&ts_language)
.map_err(|e| format!("failed to set language: {e}"))?;
let tree = parser
.parse(source, None)
.ok_or_else(|| "tree-sitter parse returned None".to_string())?;
let root = tree.root_node();
let has_error = root.has_error();
if has_error
&& matches!(language, LanguageId::C | LanguageId::Cpp)
&& source_has_backslash_crlf(source)
&& let Some(recovered) = recover_c_family_backslash_crlf(source, language)
{
return Ok(recovered);
}
let symbols = languages::extract_symbols(&root, source, language);
let (references, alias_map) = xref::extract_references(&root, source, language, is_tsx);
let diagnostic = if has_error {
collect_deepest_error_node(&root, source).map(|(message, line, column, span)| {
ParseDiagnostic {
parser: "tree-sitter".to_string(),
message: augment_message_with_nul_hint(message, source.as_bytes()),
line: Some(line),
column: Some(column),
byte_span: Some(span),
fallback_used: false,
}
})
} else {
None
};
Ok((symbols, has_error, diagnostic, references, alias_map))
}
fn source_has_backslash_crlf(source: &str) -> bool {
let b = source.as_bytes();
b.windows(3).any(|w| w == [b'\\', b'\r', b'\n'])
}
fn recover_c_family_backslash_crlf(
source: &str,
language: &LanguageId,
) -> Option<ParseSourceOutput> {
let bytes = source.as_bytes();
let mut buf = bytes.to_vec();
let mut i = 0usize;
while i + 2 < buf.len() {
if buf[i] == b'\\' && buf[i + 1] == b'\r' && buf[i + 2] == b'\n' {
buf[i + 1] = b'\n';
buf[i + 2] = b'\r';
i += 3;
} else {
i += 1;
}
}
let rewritten = String::from_utf8_lossy(&buf);
let output = panic::catch_unwind(|| parse_source(&rewritten, language, false))
.ok()?
.ok()?;
if output.1 {
return None;
}
Some(output)
}
fn count_error_nodes(root: &Node) -> usize {
let mut cursor = root.walk();
let mut stack = vec![*root];
let mut count = 0usize;
while let Some(node) = stack.pop() {
if node.is_error() || node.is_missing() {
count += 1;
}
cursor.reset(node);
if cursor.goto_first_child() {
stack.push(cursor.node());
while cursor.goto_next_sibling() {
stack.push(cursor.node());
}
}
}
count
}
fn parse_source_scored(source: &str, language: &LanguageId) -> Option<(ParseSourceOutput, usize)> {
let mut parser = Parser::new();
let ts_language = match language {
LanguageId::C => tree_sitter_c::LANGUAGE.into(),
LanguageId::Cpp => tree_sitter_cpp::LANGUAGE.into(),
_ => return None,
};
parser.set_language(&ts_language).ok()?;
let tree = parser.parse(source, None)?;
let error_nodes = count_error_nodes(&tree.root_node());
let output = parse_source(source, language, false).ok()?;
Some((output, error_nodes))
}
fn resolve_c_header_parse(
source: &str,
) -> (
LanguageId,
std::thread::Result<Result<ParseSourceOutput, String>>,
) {
let header_kind = classify_c_header_content(source);
if matches!(header_kind, CHeaderContent::Cpp) {
let output = panic::catch_unwind(|| parse_source(source, &LanguageId::Cpp, false));
return (LanguageId::Cpp, output);
}
let c_scored = panic::catch_unwind(|| parse_source_scored(source, &LanguageId::C));
let c_errors = match &c_scored {
Ok(Some((_, errors))) => *errors,
_ => {
let output = panic::catch_unwind(|| parse_source(source, &LanguageId::C, false));
return (LanguageId::C, output);
}
};
if c_errors == 0 {
return (
LanguageId::C,
c_scored.map(|opt| Ok(opt.expect("checked Some above").0)),
);
}
if matches!(header_kind, CHeaderContent::ObjectiveC) {
return (
LanguageId::C,
c_scored.map(|opt| Ok(opt.expect("checked Some above").0)),
);
}
let cpp_scored = panic::catch_unwind(|| parse_source_scored(source, &LanguageId::Cpp));
match cpp_scored {
Ok(Some((cpp_output, cpp_errors))) if cpp_errors < c_errors => {
(LanguageId::Cpp, Ok(Ok(cpp_output)))
}
_ => (
LanguageId::C,
c_scored.map(|opt| Ok(opt.expect("checked Some above").0)),
),
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum CHeaderContent {
Cpp,
ObjectiveC,
C,
}
fn classify_c_header_content(source: &str) -> CHeaderContent {
let bytes = source.as_bytes();
let mut found_cpp = false;
let mut found_objc = false;
let mut i = 0usize;
let n = bytes.len();
let is_ident = |b: u8| b.is_ascii_alphanumeric() || b == b'_';
let word_at = |start: usize, kw: &[u8]| -> bool {
let end = start + kw.len();
if end > n || &bytes[start..end] != kw {
return false;
}
let before_ok = start == 0 || !is_ident(bytes[start - 1]);
let after_ok = end == n || !is_ident(bytes[end]);
before_ok && after_ok
};
while i < n {
let b = bytes[i];
if b == b'/' && i + 1 < n && bytes[i + 1] == b'/' {
i += 2;
while i < n && bytes[i] != b'\n' {
i += 1;
}
continue;
}
if b == b'/' && i + 1 < n && bytes[i + 1] == b'*' {
i += 2;
while i + 1 < n && !(bytes[i] == b'*' && bytes[i + 1] == b'/') {
i += 1;
}
i = (i + 2).min(n);
continue;
}
if b == b'"' || b == b'\'' {
let quote = b;
i += 1;
while i < n {
if bytes[i] == b'\\' {
i += 2;
continue;
}
if bytes[i] == quote {
i += 1;
break;
}
i += 1;
}
continue;
}
if b == b':' && i + 1 < n && bytes[i + 1] == b':' {
found_cpp = true;
i += 2;
continue;
}
if b == b'@' {
for kw in [
b"interface".as_slice(),
b"implementation",
b"protocol",
b"property",
b"end",
] {
if word_at(i + 1, kw) {
found_objc = true;
break;
}
}
i += 1;
continue;
}
if b == b'#' {
let mut j = i + 1;
while j < n && (bytes[j] == b' ' || bytes[j] == b'\t') {
j += 1;
}
if word_at(j, b"import") {
found_objc = true;
}
i += 1;
continue;
}
if is_ident(b) && (i == 0 || !is_ident(bytes[i - 1])) {
if word_at(i, b"class")
|| word_at(i, b"namespace")
|| word_at(i, b"template")
|| word_at(i, b"public") && bytes.get(i + 6).is_some_and(|&c| c == b':')
|| word_at(i, b"private") && bytes.get(i + 7).is_some_and(|&c| c == b':')
|| word_at(i, b"protected") && bytes.get(i + 9).is_some_and(|&c| c == b':')
{
found_cpp = true;
}
i += 1;
while i < n && is_ident(bytes[i]) {
i += 1;
}
continue;
}
i += 1;
}
if found_cpp {
CHeaderContent::Cpp
} else if found_objc {
CHeaderContent::ObjectiveC
} else {
CHeaderContent::C
}
}
#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
enum ExpectedPartialCheck {
TsImportTypeArray,
AngularControlFlow,
}
const EXPECTED_PARTIAL_MEMO_CAP: usize = 4096;
static EXPECTED_PARTIAL_MEMO: std::sync::LazyLock<std::sync::Mutex<ExpectedPartialMemoMap>> =
std::sync::LazyLock::new(|| std::sync::Mutex::new(ExpectedPartialMemoMap::new()));
type ExpectedPartialMemoKey = (ExpectedPartialCheck, u64, u64);
type ExpectedPartialMemoMap = std::collections::HashMap<ExpectedPartialMemoKey, bool>;
fn expected_partial_memo(
check: ExpectedPartialCheck,
content: &[u8],
compute: impl FnOnce() -> bool,
) -> bool {
use std::hash::{Hash, Hasher};
let mut hasher = std::collections::hash_map::DefaultHasher::new();
content.hash(&mut hasher);
let key = (check, content.len() as u64, hasher.finish());
if let Some(&verdict) = EXPECTED_PARTIAL_MEMO
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.get(&key)
{
return verdict;
}
let verdict = compute();
let mut memo = EXPECTED_PARTIAL_MEMO
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if memo.len() >= EXPECTED_PARTIAL_MEMO_CAP {
memo.clear();
}
memo.insert(key, verdict);
verdict
}
static IMPORT_TYPE_ARRAY_SUFFIX_RE: std::sync::LazyLock<regex::Regex> =
std::sync::LazyLock::new(|| {
regex::Regex::new(
r#"(import\s*\(\s*['"][^'"]+['"]\s*\)\s*\.\s*[A-Za-z_$][A-Za-z0-9_$]*)(?:\s*\[\s*\])+"#,
)
.expect("SF-003 import-type-array regex is a valid pattern")
});
pub(crate) fn is_expected_typescript_import_type_array_limitation(
language: &LanguageId,
content: &[u8],
is_tsx: bool,
) -> bool {
if !matches!(language, LanguageId::TypeScript) {
return false;
}
expected_partial_memo(ExpectedPartialCheck::TsImportTypeArray, content, || {
let source = String::from_utf8_lossy(content);
if !IMPORT_TYPE_ARRAY_SUFFIX_RE.is_match(&source) {
return false;
}
let original_has_error =
match panic::catch_unwind(|| parse_source(&source, language, is_tsx)) {
Ok(Ok((_, has_error, _, _, _))) => has_error,
_ => return false,
};
if !original_has_error {
return false;
}
let neutralized = IMPORT_TYPE_ARRAY_SUFFIX_RE.replace_all(&source, "${1} ");
match panic::catch_unwind(|| parse_source(&neutralized, language, is_tsx)) {
Ok(Ok((_, has_error, _, _, _))) => !has_error,
_ => false,
}
})
}
static ANGULAR_CONTROL_FLOW_OPENER_RE: std::sync::LazyLock<regex::Regex> =
std::sync::LazyLock::new(|| {
regex::Regex::new(r"(?m)(^|[^A-Za-z0-9_$])(@(?:if|for|switch|defer|else\s+if)\s*\()")
.expect("SF-004 Angular control-flow opener regex is a valid pattern")
});
pub(crate) fn is_expected_angular_template_control_flow_limitation(
language: &LanguageId,
content: &[u8],
) -> bool {
if !matches!(language, LanguageId::Html) {
return false;
}
expected_partial_memo(ExpectedPartialCheck::AngularControlFlow, content, || {
let source = String::from_utf8_lossy(content);
if !ANGULAR_CONTROL_FLOW_OPENER_RE.is_match(&source) {
return false;
}
let original_has_error =
match panic::catch_unwind(|| parse_source(&source, language, false)) {
Ok(Ok((_, has_error, _, _, _))) => has_error,
_ => return false,
};
if !original_has_error {
return false;
}
let neutralized = neutralize_angular_control_flow_operators(&source);
match panic::catch_unwind(|| parse_source(&neutralized, language, false)) {
Ok(Ok((_, has_error, _, _, _))) => !has_error,
_ => false,
}
})
}
fn neutralize_angular_control_flow_operators(source: &str) -> String {
let original = source.as_bytes();
let mut buf = original.to_vec();
for m in ANGULAR_CONTROL_FLOW_OPENER_RE.find_iter(source) {
let open_paren = m.end();
debug_assert_eq!(original.get(open_paren - 1), Some(&b'('));
let mut depth: usize = 1;
let mut i = open_paren;
let mut closed = false;
let mut neutralized_any = false;
while i < original.len() {
match original[i] {
b'(' => depth += 1,
b')' => {
depth -= 1;
if depth == 0 {
closed = true;
break;
}
}
b'<' | b'>' => {
buf[i] = b' ';
neutralized_any = true;
}
_ => {}
}
i += 1;
}
if !closed && neutralized_any {
buf[open_paren..i].copy_from_slice(&original[open_paren..i]);
}
}
String::from_utf8(buf).unwrap_or_else(|_| source.to_string())
}
pub fn extract_symbols_for_diff(source: &str, path: &str) -> Option<Vec<(String, String)>> {
let ext = path.rsplit('.').next().unwrap_or("");
let language = LanguageId::from_extension(ext)?;
if config_extractors::is_config_language(&language) {
return None; }
let is_tsx = LanguageId::is_tsx_path(path);
let result = panic::catch_unwind(|| parse_source(source, &language, is_tsx));
let (symbols, ..) = match result {
Ok(Ok(output)) => output,
_ => return None,
};
let pairs: Vec<(String, String)> = symbols
.iter()
.map(|sym| {
let (start, end) = sym.byte_range;
let body = &source[start as usize..end as usize];
(sym.name.clone(), crate::hash::digest_hex(body.as_bytes()))
})
.collect();
Some(pairs)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::domain::{FileOutcome, LanguageId, SymbolKind};
#[test]
fn test_classify_c_header_content_cpp_markers() {
for src in [
"class FlutterWindow {};",
"namespace ns { }",
"template <typename T> struct S {};",
"flutter::DartProject project_;",
"class A : public B {\n public:\n};",
] {
assert_eq!(
classify_c_header_content(src),
CHeaderContent::Cpp,
"expected Cpp for: {src:?}"
);
}
}
#[test]
fn test_classify_c_header_content_plain_c() {
for src in [
"struct point { int x; int y; };",
"int add(int a, int b);",
"#define MAX 10\nvoid f(void);",
"typedef struct node node_t;",
] {
assert_eq!(
classify_c_header_content(src),
CHeaderContent::C,
"expected C for: {src:?}"
);
}
}
#[test]
fn test_classify_c_header_content_objc_markers() {
for src in [
"@interface Foo : NSObject\n@end",
"#import <Foundation/Foundation.h>",
"@protocol Bar\n@end",
"@property(nonatomic) int x;",
] {
assert_eq!(
classify_c_header_content(src),
CHeaderContent::ObjectiveC,
"expected ObjectiveC for: {src:?}"
);
}
}
#[test]
fn test_classify_c_header_content_ignores_comments_and_strings() {
assert_eq!(
classify_c_header_content("// this header defines a class\nint f(void);"),
CHeaderContent::C,
"`class` in a line comment must not trigger Cpp"
);
assert_eq!(
classify_c_header_content("/* namespace std */\nint f(void);"),
CHeaderContent::C,
"`namespace` in a block comment must not trigger Cpp"
);
assert_eq!(
classify_c_header_content("const char* s = \"a::b\";\nint f(void);"),
CHeaderContent::C,
"`::` inside a string literal must not trigger Cpp"
);
assert_eq!(
classify_c_header_content("int subclass; void classify(void);"),
CHeaderContent::C,
"`class` as a substring of an identifier must not trigger Cpp"
);
}
#[test]
fn test_classify_c_header_content_cpp_wins_over_objc() {
assert_eq!(
classify_c_header_content("#import <F/F.h>\nclass Foo {};"),
CHeaderContent::Cpp
);
}
#[test]
fn test_process_file_rust_extracts_function() {
let source = b"fn hello() { }";
let result = process_file("test.rs", source, LanguageId::Rust);
assert_eq!(result.outcome, FileOutcome::Processed);
assert!(!result.symbols.is_empty());
assert_eq!(result.symbols[0].name, "hello");
assert_eq!(result.symbols[0].kind, SymbolKind::Function);
}
#[test]
fn test_process_file_rust_accepts_borrowed_raw_identifier() {
let source = b"fn main() { let raw = 1; let _x = &raw; }";
let result = process_file("test.rs", source, LanguageId::Rust);
assert_eq!(result.outcome, FileOutcome::Processed);
}
#[test]
fn test_process_file_rust_preserves_raw_borrow_syntax() {
for source in [
b"fn main() { let value = 1; let _ptr = &raw const value; }" as &[u8],
b"fn main() { let mut value = 1; let _ptr = &raw mut value; }",
] {
let result = process_file("test.rs", source, LanguageId::Rust);
assert_eq!(result.outcome, FileOutcome::Processed);
}
}
#[test]
fn test_process_file_python_extracts_function() {
let source = b"def greet():\n pass";
let result = process_file("test.py", source, LanguageId::Python);
assert_eq!(result.outcome, FileOutcome::Processed);
assert!(!result.symbols.is_empty());
assert_eq!(result.symbols[0].name, "greet");
assert_eq!(result.symbols[0].kind, SymbolKind::Function);
}
#[test]
fn test_process_file_javascript_extracts_function() {
let source = b"function doStuff() { }";
let result = process_file("test.js", source, LanguageId::JavaScript);
assert_eq!(result.outcome, FileOutcome::Processed);
assert!(!result.symbols.is_empty());
assert_eq!(result.symbols[0].name, "doStuff");
assert_eq!(result.symbols[0].kind, SymbolKind::Function);
}
#[test]
fn test_process_file_jsx_parses_clean_and_extracts_symbols() {
let source = br#"
import { Row } from './Row';
export function Table({ rows }) {
return (
<>
{rows.map((r) => (
<Row key={r.id} value={r.value} />
))}
</>
);
}
"#;
let result = process_file("src/Table.jsx", source, LanguageId::JavaScript);
assert_eq!(
result.outcome,
FileOutcome::Processed,
"a `.jsx` file must parse cleanly under the JavaScript grammar; got {:?} / {:?}",
result.outcome,
result.parse_diagnostic
);
assert!(
result.parse_diagnostic.is_none(),
"clean JSX parse must not attach a diagnostic; got {:?}",
result.parse_diagnostic
);
assert!(
result.symbols.iter().any(|s| s.name == "Table"),
"the JSX component function `Table` must be extracted; got {:?}",
result.symbols
);
}
#[test]
fn test_process_file_typescript_extracts_interface() {
let source = b"interface Greeter { greet(): void; }";
let result = process_file("test.ts", source, LanguageId::TypeScript);
assert_eq!(result.outcome, FileOutcome::Processed);
let interface = result
.symbols
.iter()
.find(|s| s.kind == SymbolKind::Interface);
assert!(interface.is_some());
assert_eq!(interface.unwrap().name, "Greeter");
}
#[test]
fn test_sf003_class_field_import_type_array_is_expected_limitation() {
let source = b"class C { private subs: import('rxjs').Subscription[] = []; }";
let result = process_file(
"workflow-builder.component.ts",
source,
LanguageId::TypeScript,
);
assert!(
matches!(result.outcome, FileOutcome::PartialParse { .. }),
"expected grammar still reports a partial parse for the limitation case"
);
assert!(
is_expected_typescript_import_type_array_limitation(
&LanguageId::TypeScript,
source,
false
),
"class-field import-type-array must be recognized as an expected grammar limitation"
);
assert!(
result.symbols.iter().any(|s| s.name == "C"),
"class symbol C must still be extracted despite the partial parse"
);
}
#[test]
fn test_sf003_type_alias_import_type_array_is_expected_limitation() {
let source = b"type S = import('rxjs').Subscription[];";
let result = process_file("types.ts", source, LanguageId::TypeScript);
assert!(
matches!(result.outcome, FileOutcome::PartialParse { .. }),
"type-alias import-type-array still reports a partial parse"
);
assert!(
is_expected_typescript_import_type_array_limitation(
&LanguageId::TypeScript,
source,
false
),
"type-alias import-type-array must be recognized as an expected grammar limitation"
);
}
#[test]
fn test_sf003_multidim_import_type_array_is_expected_limitation() {
let source = b"type S = import('rxjs').Subscription[][];";
assert!(
is_expected_typescript_import_type_array_limitation(
&LanguageId::TypeScript,
source,
false
),
"multi-dimensional import-type-array must be recognized as an expected limitation"
);
}
#[test]
fn test_expected_partial_memo_hit_skips_recompute() {
let content = b"memo-probe :: post-v7.19.0 review fix 1 :: unique";
assert!(expected_partial_memo(
ExpectedPartialCheck::TsImportTypeArray,
content,
|| true
));
assert!(expected_partial_memo(
ExpectedPartialCheck::TsImportTypeArray,
content,
|| panic!("verdict must come from the memo, not a recompute"),
));
assert!(!expected_partial_memo(
ExpectedPartialCheck::AngularControlFlow,
content,
|| false
));
let other = b"memo-probe :: post-v7.19.0 review fix 1 :: uniquf";
assert_eq!(content.len(), other.len());
assert!(!expected_partial_memo(
ExpectedPartialCheck::TsImportTypeArray,
other,
|| false
));
}
#[test]
fn test_sf003_negative_control_genuinely_broken_array_stays_partial() {
let source = b"class C { private subs: import('rxjs').Subscription[] = [ ; foo bar baz }";
let result = process_file("broken.ts", source, LanguageId::TypeScript);
assert!(
matches!(result.outcome, FileOutcome::PartialParse { .. }),
"the genuinely broken file is a partial parse"
);
assert!(
!is_expected_typescript_import_type_array_limitation(
&LanguageId::TypeScript,
source,
false
),
"a genuinely broken file sharing the import-type-array prefix MUST stay partial"
);
}
#[test]
fn test_sf003_negative_control_identifier_glue_stays_partial() {
let source = b"type S = import('x').Sub[]scription;";
let result = process_file("glue-type-alias.ts", source, LanguageId::TypeScript);
assert!(
matches!(result.outcome, FileOutcome::PartialParse { .. }),
"the genuinely broken identifier-glue file is a partial parse"
);
assert!(
!is_expected_typescript_import_type_array_limitation(
&LanguageId::TypeScript,
source,
false
),
"deleting `[]` would glue `Sub[]scription` into a valid identifier; the \
token-preserving neutralization must keep this broken file partial"
);
}
#[test]
fn test_sf003_negative_control_identifier_glue_class_field_stays_partial() {
let source = b"class C { private a: import('rxjs').Sub[]scription = x; }";
let result = process_file("glue-class-field.ts", source, LanguageId::TypeScript);
assert!(
matches!(result.outcome, FileOutcome::PartialParse { .. }),
"the genuinely broken class-field identifier-glue file is a partial parse"
);
assert!(
!is_expected_typescript_import_type_array_limitation(
&LanguageId::TypeScript,
source,
false
),
"a class-field import-type member glued to a trailing identifier fragment \
must stay partial under the token-preserving neutralization"
);
}
#[test]
fn test_sf003_negative_control_unrelated_syntax_error_stays_partial() {
let source = b"class C { private x: = ; }";
assert!(
!is_expected_typescript_import_type_array_limitation(
&LanguageId::TypeScript,
source,
false
),
"an unrelated syntax error must not be excused as an import-type-array limitation"
);
}
#[test]
fn test_sf003_negative_control_valid_array_plus_real_error_stays_partial() {
let source = b"type S = import('rxjs').Subscription[]; class C { private x: = ; }";
assert!(
!is_expected_typescript_import_type_array_limitation(
&LanguageId::TypeScript,
source,
false
),
"a real error elsewhere must keep the file partial even when an import-type-array is present"
);
}
#[test]
fn test_sf003_detector_is_typescript_gated() {
let source = b"type S = import('rxjs').Subscription[];";
assert!(
!is_expected_typescript_import_type_array_limitation(
&LanguageId::JavaScript,
source,
false
),
"the import-type-array limitation detector must be TypeScript-gated"
);
}
#[test]
fn test_sf003_detector_false_on_clean_typescript() {
let source = b"type S = import('rxjs').Subscription;";
assert!(
!is_expected_typescript_import_type_array_limitation(
&LanguageId::TypeScript,
source,
false
),
"clean scalar import-type must not be classified as a limitation"
);
}
#[test]
fn test_tsx_jsx_component_parses_clean_and_extracts_symbols() {
let source = br#"
export function App() {
return (
<QueryClientProvider client={queryClient}>
<RouterProvider router={router} />
</QueryClientProvider>
);
}
"#;
let result = process_file("src/App.tsx", source, LanguageId::TypeScript);
assert_eq!(
result.outcome,
FileOutcome::Processed,
"a `.tsx` JSX component must parse cleanly under the TSX grammar; got {:?} / {:?}",
result.outcome,
result.parse_diagnostic
);
assert!(
result.parse_diagnostic.is_none(),
"clean TSX parse must not attach a diagnostic; got {:?}",
result.parse_diagnostic
);
assert!(
result.symbols.iter().any(|s| s.name == "App"),
"the JSX component function `App` must be extracted; got {:?}",
result.symbols
);
}
#[test]
fn test_tsx_grammar_selected_by_extension_not_languageid() {
let source = br#"
export function App() {
return <div>hi</div>;
}
"#;
let as_tsx = process_file("src/App.tsx", source, LanguageId::TypeScript);
assert_eq!(
as_tsx.outcome,
FileOutcome::Processed,
"JSX under `.tsx` must parse clean"
);
let as_ts = process_file("src/App.ts", source, LanguageId::TypeScript);
assert!(
matches!(as_ts.outcome, FileOutcome::PartialParse { .. }),
"JSX under `.ts` must remain a partial parse (plain TS grammar has no JSX); got {:?}",
as_ts.outcome
);
}
#[test]
fn test_ts_angle_bracket_type_assertion_still_parses_clean() {
let source = b"const y: unknown = 1;\nconst x = <number>y;\n";
let result = process_file("src/cast.ts", source, LanguageId::TypeScript);
assert_eq!(
result.outcome,
FileOutcome::Processed,
"a `.ts` angle-bracket cast must parse cleanly under the TypeScript grammar; got {:?} / {:?}",
result.outcome,
result.parse_diagnostic
);
assert!(
result.parse_diagnostic.is_none(),
"clean `.ts` cast must not attach a diagnostic; got {:?}",
result.parse_diagnostic
);
assert!(
result.symbols.iter().any(|s| s.name == "x"),
"the cast binding `x` must be extracted; got {:?}",
result.symbols
);
}
#[test]
fn test_is_tsx_path_classifier() {
assert!(LanguageId::is_tsx_path("src/App.tsx"));
assert!(LanguageId::is_tsx_path("App.TSX"));
assert!(LanguageId::is_tsx_path(r"src\components\App.tsx"));
assert!(!LanguageId::is_tsx_path("src/App.ts"));
assert!(!LanguageId::is_tsx_path("src/App.jsx"));
assert!(!LanguageId::is_tsx_path("tsx"));
assert!(!LanguageId::is_tsx_path("dir.tsx/file.ts"));
assert!(!LanguageId::is_tsx_path("noext"));
}
#[test]
fn test_process_file_go_extracts_function() {
let source = b"package main\nfunc main() { }";
let result = process_file("test.go", source, LanguageId::Go);
assert_eq!(result.outcome, FileOutcome::Processed);
assert!(!result.symbols.is_empty());
assert_eq!(result.symbols[0].name, "main");
assert_eq!(result.symbols[0].kind, SymbolKind::Function);
}
#[test]
fn test_process_file_partial_parse() {
let source = b"fn broken( { }";
let result = process_file("bad.rs", source, LanguageId::Rust);
assert!(matches!(result.outcome, FileOutcome::PartialParse { .. }));
}
#[test]
fn test_process_file_partial_parse_has_diagnostic() {
let source = b"fn broken( { }";
let result = process_file("bad.rs", source, LanguageId::Rust);
assert!(matches!(result.outcome, FileOutcome::PartialParse { .. }));
let diag = result
.parse_diagnostic
.expect("should have a diagnostic for partial parse");
assert_eq!(diag.parser, "tree-sitter");
assert!(diag.line.is_some(), "diagnostic should have a line number");
assert!(
diag.column.is_some(),
"diagnostic should have a column number"
);
assert!(
diag.byte_span.is_some(),
"diagnostic should have a byte span"
);
assert!(
diag.message.contains("syntax"),
"message should describe the error"
);
}
#[test]
fn test_process_file_partial_parse_diagnostic_pins_location() {
let source = b"fn foo() {}\nfn bar() {}\nfn broken( { }";
let result = process_file("multi.rs", source, LanguageId::Rust);
assert!(
matches!(result.outcome, FileOutcome::PartialParse { .. }),
"source with a line-3 syntax error should be partial-parsed; got {:?}",
result.outcome
);
let diag = result
.parse_diagnostic
.expect("partial parse must attach a ParseDiagnostic");
assert_eq!(diag.parser, "tree-sitter");
assert!(
!diag.fallback_used,
"tree-sitter parses must not set fallback_used; that flag is reserved \
for config extractors that recover via a secondary parser"
);
let line = diag.line.expect("diagnostic must carry a line number");
assert!(
line >= 3,
"error is on line 3 of the source; diagnostic reported line {line}"
);
let column = diag.column.expect("diagnostic must carry a column number");
assert!(column >= 1, "columns are 1-based; got {column}");
let (span_start, span_end) = diag
.byte_span
.expect("diagnostic must carry a byte span for downstream editors");
assert!(
span_start <= span_end,
"byte_span must be ordered; got {span_start}..{span_end}"
);
assert!(
(span_end as usize) <= source.len(),
"byte_span must fit inside source (len {}); got end {span_end}",
source.len()
);
assert!(
span_start >= 24,
"byte_span should point at line 3 content (starts at byte 24); got {span_start}"
);
let loc = diag
.location_display()
.expect("location_display must render when both line and column are present");
assert!(
loc.contains(&format!("line {line}")),
"location_display must include the line; got {loc}"
);
assert!(
loc.contains(&format!("column {column}")),
"location_display must include the column; got {loc}"
);
let summary = diag.summary();
assert!(
summary.contains("tree-sitter:"),
"summary should prefix with parser name; got {summary}"
);
assert!(
summary.contains(&format!("line {line}")),
"summary should include location for downstream display; got {summary}"
);
}
#[test]
fn test_parse_source_reports_deepest_actionable_nested_error_node() {
let source =
"def compute():\n value = outer(inner(1 + 2, tail)\n other = {'a': {'b': 1}}\n";
let (_symbols, has_error, diagnostic, _references, _aliases) =
parse_source(source, &LanguageId::Python, false).expect("python parse should complete");
assert!(has_error, "fixture must contain a tree-sitter parse error");
let diag = diagnostic.expect("partial parse must attach a diagnostic");
assert_eq!(diag.parser, "tree-sitter");
assert_eq!(diag.line, Some(2), "unexpected diagnostic: {diag:?}");
assert_eq!(diag.column, Some(19), "unexpected diagnostic: {diag:?}");
assert_eq!(
diag.byte_span,
Some((33, 51)),
"diagnostic must pin the nested call error, not the outer assignment; got {diag:?}"
);
assert!(
diag.message
.contains("syntax error near `inner(1 + 2, tail)`"),
"diagnostic should include the nested call context; got {diag:?}"
);
}
#[test]
fn test_parse_source_reports_actionable_missing_node_context() {
let source = "fn compute() {\n let value = outer(inner(1 + ), tail);\n}\n";
let (_symbols, has_error, diagnostic, _references, _aliases) =
parse_source(source, &LanguageId::Rust, false).expect("rust parse should complete");
assert!(has_error, "fixture must contain a tree-sitter parse error");
let diag = diagnostic.expect("partial parse must attach a diagnostic");
assert_eq!(diag.parser, "tree-sitter");
assert_eq!(diag.line, Some(2), "unexpected diagnostic: {diag:?}");
assert_eq!(diag.column, Some(32), "unexpected diagnostic: {diag:?}");
assert_eq!(
diag.byte_span,
Some((46, 46)),
"diagnostic must pin the zero-width missing node; got {diag:?}"
);
assert!(
diag.message.contains("syntax missing identifier"),
"diagnostic should name the missing node kind; got {diag:?}"
);
assert!(
diag.message.contains("), tail"),
"diagnostic should include source context after the missing node; got {diag:?}"
);
}
#[test]
fn test_process_file_computes_content_hash() {
let source = b"fn foo() {}";
let result = process_file("hash_test.rs", source, LanguageId::Rust);
assert!(!result.content_hash.is_empty());
assert_eq!(result.content_hash, digest_hex(source));
}
#[test]
fn test_process_file_byte_len() {
let source = b"fn bar() {}";
let result = process_file("len.rs", source, LanguageId::Rust);
assert_eq!(result.byte_len, source.len() as u64);
}
#[test]
fn test_process_file_preserves_relative_path() {
let result = process_file("src/lib.rs", b"fn x() {}", LanguageId::Rust);
assert_eq!(result.relative_path, "src/lib.rs");
}
#[test]
fn test_process_file_never_panics_on_adversarial_input() {
let cases: &[(&[u8], &str, LanguageId)] = &[
(b"\xff\xfe\x00\x01", "binary.rs", LanguageId::Rust),
(b"", "empty.py", LanguageId::Python),
(&[0u8; 10000], "zeros.js", LanguageId::JavaScript),
(b"\n\n\n\n\n", "newlines.ts", LanguageId::TypeScript),
("\u{200b}\u{200b}".as_bytes(), "zwsp.go", LanguageId::Go),
(
b"\0\0\0fn main() {}\0\0",
"null_padded.rs",
LanguageId::Rust,
),
];
for &(source, path, ref lang) in cases {
let result = process_file(path, source, lang.clone());
assert_eq!(result.relative_path, path);
assert_eq!(result.byte_len, source.len() as u64);
assert!(!result.content_hash.is_empty());
}
}
#[test]
fn test_process_file_ruby_extracts_method() {
let source = b"def hello\n puts 'hi'\nend";
let result = process_file("app.rb", source, LanguageId::Ruby);
assert_eq!(result.outcome, FileOutcome::Processed);
assert!(
!result.symbols.is_empty(),
"should have symbols for Ruby source"
);
}
#[test]
fn test_process_file_is_idempotent_across_re_parses() {
let rust_src: &[u8] = b"use std::collections::HashMap;\n\
use std::fmt::Display;\n\
\n\
pub struct Cache<K, V> {\n\
store: HashMap<K, V>,\n\
}\n\
\n\
impl<K: std::hash::Hash + Eq, V> Cache<K, V> {\n\
pub fn new() -> Self { Self { store: HashMap::new() } }\n\
pub fn insert(&mut self, k: K, v: V) -> Option<V> { self.store.insert(k, v) }\n\
}\n\
\n\
pub fn make() -> Cache<String, u32> { Cache::new() }\n";
let python_src: &[u8] = b"import os\n\
import sys\n\
\n\
class Greeter:\n\
def __init__(self, name):\n\
self.name = name\n\
\n\
def greet(self):\n\
return f\"hello {self.name}\"\n\
\n\
def main():\n\
Greeter(\"world\").greet()\n";
let ts_src: &[u8] = b"interface Greeter { greet(): string; }\n\
export class Hello implements Greeter {\n\
constructor(private name: string) {}\n\
greet() { return `hi ${this.name}`; }\n\
}\n";
let broken_src: &[u8] = b"fn broken( { }";
let cases: &[(&[u8], &str, LanguageId)] = &[
(rust_src, "cache.rs", LanguageId::Rust),
(python_src, "greet.py", LanguageId::Python),
(ts_src, "hello.ts", LanguageId::TypeScript),
(broken_src, "broken.rs", LanguageId::Rust),
];
for &(source, path, ref lang) in cases {
let first = process_file(path, source, lang.clone());
let second = process_file(path, source, lang.clone());
let third = process_file(path, source, lang.clone());
assert_eq!(
first, second,
"{path}: identical bytes must yield identical FileProcessingResult (run 1 vs 2)"
);
assert_eq!(
second, third,
"{path}: identical bytes must yield identical FileProcessingResult (run 2 vs 3)"
);
}
}
#[test]
fn test_parse_source_zero_bytes() {
let result = parse_source("", &LanguageId::Rust, false)
.expect("parse_source must handle empty input without error");
let (symbols, _has_error, _diagnostic, references, alias_map) = result;
assert!(symbols.is_empty(), "empty source has no symbols");
assert!(references.is_empty(), "empty source has no references");
assert!(alias_map.is_empty(), "empty source has no aliases");
}
#[test]
fn test_parse_source_null_bytes_only() {
let source: String = "\0".repeat(4096);
let result = parse_source(&source, &LanguageId::Rust, false)
.expect("parse_source must handle null-byte input without error");
let (_symbols, _has_error, _diagnostic, _references, _aliases) = result;
}
#[test]
fn test_augment_message_with_nul_hint_appends_for_nul_source() {
let augmented = augment_message_with_nul_hint(
"syntax error near ` `".to_string(),
b"const a = `x\0y`;\n",
);
assert!(
augmented.contains("file contains 1 NUL byte at offset(s) 12"),
"hint must name count and offset; got: {augmented}"
);
assert!(
augmented.contains("tree-sitter cannot parse NUL in source"),
"hint must explain the real cause; got: {augmented}"
);
assert!(augmented.starts_with("syntax error near ` `"));
}
#[test]
fn test_augment_message_with_nul_hint_noop_for_clean_source() {
let original = "syntax error near `oops`".to_string();
let augmented = augment_message_with_nul_hint(original.clone(), b"fn main() {}\n");
assert_eq!(augmented, original, "clean source must not gain a NUL hint");
}
#[test]
fn test_parse_source_diagnostic_carries_nul_hint_when_present() {
let source = "function mint() {\n const token = `prefix\0suffix`;\n return token;\n}\n";
let (_symbols, has_error, diagnostic, _refs, _aliases) =
parse_source(source, &LanguageId::JavaScript, false)
.expect("parse_source must not crash on a NUL-bearing template literal");
if has_error && let Some(diag) = diagnostic {
assert!(
diag.message
.contains("tree-sitter cannot parse NUL in source"),
"diagnostic must carry the NUL hint; got: {}",
diag.message
);
}
}
#[test]
fn test_parse_source_wide_multibyte_error_region_no_panic() {
let source: String = "€".repeat(100);
parse_source(&source, &LanguageId::Rust, false)
.expect("parse_source must not panic on wide multi-byte error region");
}
#[test]
fn test_parse_source_mixed_multibyte_error_boundary_no_panic() {
let mut source = String::new();
source.push_str("struct S { ");
for _ in 0..14 {
source.push('€');
}
source.push(' ');
parse_source(&source, &LanguageId::Rust, false)
.expect("parse_source must not panic when error snippet spans multi-byte chars");
}
#[test]
fn test_process_file_deeply_nested_expression_no_stack_blow() {
const DEPTH: usize = 10_000;
let handle = std::thread::Builder::new()
.stack_size(16 * 1024 * 1024)
.spawn(|| {
let mut source = String::with_capacity(DEPTH * 2 + 32);
source.push_str("fn f() -> i32 { ");
for _ in 0..DEPTH {
source.push('(');
}
source.push('1');
for _ in 0..DEPTH {
source.push(')');
}
source.push_str(" }");
let result = process_file("deep.rs", source.as_bytes(), LanguageId::Rust);
assert_eq!(result.byte_len, source.len() as u64);
assert!(!result.content_hash.is_empty());
})
.expect("spawn stack-blow probe thread");
handle.join().expect("deep-nesting probe must not panic");
}
#[test]
fn test_process_file_deeply_nested_expression_on_default_stack_no_panic() {
const DEPTH: usize = 10_000;
let mut source = String::with_capacity(DEPTH * 2 + 32);
source.push_str("fn f() -> i32 { ");
for _ in 0..DEPTH {
source.push('(');
}
source.push('1');
for _ in 0..DEPTH {
source.push(')');
}
source.push_str(" }");
let result = process_file("deep_default.rs", source.as_bytes(), LanguageId::Rust);
assert_eq!(result.byte_len, source.len() as u64);
assert!(!result.content_hash.is_empty());
}
}