pub mod ast_grep;
pub mod config_extractors;
#[cfg(test)]
mod inline_tests;
pub mod languages;
pub mod xref;
use std::collections::HashMap;
use std::panic;
use tree_sitter::Parser;
use tree_sitter::Node;
use crate::domain::{
FileClassification, FileOutcome, FileProcessingResult, LanguageId, ParseDiagnostic,
ReferenceRecord, SymbolRecord,
};
use crate::hash::digest_hex;
type ParseSourceOutput = (
Vec<SymbolRecord>,
bool,
Option<ParseDiagnostic>,
Vec<ReferenceRecord>,
HashMap<String, String>,
);
pub fn process_file(
relative_path: &str,
bytes: &[u8],
language: LanguageId,
) -> FileProcessingResult {
process_file_with_classification(
relative_path,
bytes,
language,
FileClassification::for_code_path(relative_path),
)
}
pub fn process_file_with_classification(
relative_path: &str,
bytes: &[u8],
language: LanguageId,
classification: FileClassification,
) -> FileProcessingResult {
let byte_len = bytes.len() as u64;
let content_hash = digest_hex(bytes);
if config_extractors::is_config_language(&language) {
let result = config_extractors::extractor_for(&language).map(|e| e.extract(bytes));
let (symbols, outcome, parse_diagnostic) = match result {
Some(r) => {
let (outcome, parse_diagnostic) = match r.outcome {
config_extractors::ExtractionOutcome::Ok => (FileOutcome::Processed, None),
config_extractors::ExtractionOutcome::Partial(diagnostic) => (
FileOutcome::PartialParse {
warning: diagnostic.summary(),
},
Some(diagnostic),
),
config_extractors::ExtractionOutcome::Failed(diagnostic) => (
FileOutcome::Failed {
error: diagnostic.summary(),
},
Some(diagnostic),
),
};
(r.symbols, outcome, parse_diagnostic)
}
None => (vec![], FileOutcome::Processed, None),
};
return FileProcessingResult {
relative_path: relative_path.to_string(),
language,
classification,
outcome,
parse_diagnostic,
symbols,
byte_len,
content_hash,
references: vec![],
alias_map: HashMap::new(),
};
}
let source = String::from_utf8_lossy(bytes);
let parse_result = panic::catch_unwind(|| parse_source(&source, &language));
match parse_result {
Ok(Ok((symbols, has_error, diagnostic, references, alias_map))) => {
let outcome = if has_error {
let warning = diagnostic.as_ref().map(|d| d.summary()).unwrap_or_else(|| {
"tree-sitter reported syntax errors in the parse tree".to_string()
});
FileOutcome::PartialParse { warning }
} else {
FileOutcome::Processed
};
FileProcessingResult {
relative_path: relative_path.to_string(),
language,
classification,
outcome,
parse_diagnostic: diagnostic,
symbols,
byte_len,
content_hash,
references,
alias_map,
}
}
Ok(Err(err)) => FileProcessingResult {
relative_path: relative_path.to_string(),
language,
classification,
outcome: FileOutcome::Failed {
error: err.to_string(),
},
parse_diagnostic: None,
symbols: vec![],
byte_len,
content_hash,
references: vec![],
alias_map: HashMap::new(),
},
Err(_panic) => FileProcessingResult {
relative_path: relative_path.to_string(),
language,
classification,
outcome: FileOutcome::Failed {
error: "tree-sitter parser panicked during parsing".to_string(),
},
parse_diagnostic: None,
symbols: vec![],
byte_len,
content_hash,
references: vec![],
alias_map: HashMap::new(),
},
}
}
fn collect_deepest_error_node(root: &Node, source: &str) -> Option<(String, u32, u32, (u32, u32))> {
let mut cursor = root.walk();
let mut stack = vec![(*root, 0usize)];
let mut best: Option<(Node, usize)> = None;
while let Some((node, depth)) = stack.pop() {
if node.is_error() || node.is_missing() {
best = match best {
Some((current, current_depth))
if !error_candidate_is_better(node, depth, current, current_depth) =>
{
Some((current, current_depth))
}
_ => Some((node, depth)),
};
}
cursor.reset(node);
if cursor.goto_first_child() {
let mut children = vec![cursor.node()];
while cursor.goto_next_sibling() {
children.push(cursor.node());
}
stack.extend(children.into_iter().rev().map(|child| (child, depth + 1)));
}
}
best.map(|(node, _depth)| {
let start = node.start_position();
let snippet_start = node.start_byte().min(source.len());
let snippet_limit = snippet_start + 40;
let span_end = node.end_byte().min(source.len());
let mut snippet_end = if span_end > snippet_start {
span_end.min(snippet_limit)
} else {
snippet_limit.min(source.len())
};
while snippet_end > snippet_start && !source.is_char_boundary(snippet_end) {
snippet_end -= 1;
}
let snippet = &source[snippet_start..snippet_end];
let kind = if node.is_missing() {
format!("missing {}", node.kind())
} else {
"error".to_string()
};
let message = format!("syntax {kind} near `{}`", snippet.replace('\n', "\\n"));
(
message,
start.row as u32 + 1, start.column as u32 + 1, (node.start_byte() as u32, node.end_byte() as u32),
)
})
}
fn error_candidate_is_better(
candidate: Node,
candidate_depth: usize,
current: Node,
current_depth: usize,
) -> bool {
if candidate_depth != current_depth {
return candidate_depth > current_depth;
}
let candidate_len = candidate.end_byte().saturating_sub(candidate.start_byte());
let current_len = current.end_byte().saturating_sub(current.start_byte());
if candidate_len != current_len {
return candidate_len < current_len;
}
if candidate.start_byte() != current.start_byte() {
return candidate.start_byte() < current.start_byte();
}
candidate.is_missing() && !current.is_missing()
}
pub(crate) fn parse_source(
source: &str,
language: &LanguageId,
) -> Result<ParseSourceOutput, String> {
let mut parser = Parser::new();
let ts_language = match language {
LanguageId::Rust => tree_sitter_rust::LANGUAGE.into(),
LanguageId::Python => tree_sitter_python::LANGUAGE.into(),
LanguageId::JavaScript => tree_sitter_javascript::LANGUAGE.into(),
LanguageId::TypeScript => tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(),
LanguageId::Go => tree_sitter_go::LANGUAGE.into(),
LanguageId::Java => tree_sitter_java::LANGUAGE.into(),
LanguageId::C => tree_sitter_c::LANGUAGE.into(),
LanguageId::Cpp => tree_sitter_cpp::LANGUAGE.into(),
LanguageId::CSharp => tree_sitter_c_sharp::LANGUAGE.into(),
LanguageId::Ruby => tree_sitter_ruby::LANGUAGE.into(),
LanguageId::Php => tree_sitter_php::LANGUAGE_PHP.into(),
LanguageId::Swift => tree_sitter_swift::LANGUAGE.into(),
LanguageId::Perl => tree_sitter_perl::LANGUAGE.into(),
LanguageId::Kotlin => tree_sitter_kotlin_sg::LANGUAGE.into(),
LanguageId::Dart => tree_sitter_dart::language(),
LanguageId::Elixir => tree_sitter_elixir::LANGUAGE.into(),
LanguageId::Json
| LanguageId::Toml
| LanguageId::Yaml
| LanguageId::Markdown
| LanguageId::Env => unreachable!("config types are handled before parse_source"),
LanguageId::Html => tree_sitter_html::LANGUAGE.into(),
LanguageId::Css => tree_sitter_css::LANGUAGE.into(),
LanguageId::Scss => tree_sitter_scss::language(),
};
parser
.set_language(&ts_language)
.map_err(|e| format!("failed to set language: {e}"))?;
let tree = parser
.parse(source, None)
.ok_or_else(|| "tree-sitter parse returned None".to_string())?;
let root = tree.root_node();
let has_error = root.has_error();
let symbols = languages::extract_symbols(&root, source, language);
let (references, alias_map) = xref::extract_references(&root, source, language);
let diagnostic = if has_error {
collect_deepest_error_node(&root, source).map(|(message, line, column, span)| {
ParseDiagnostic {
parser: "tree-sitter".to_string(),
message,
line: Some(line),
column: Some(column),
byte_span: Some(span),
fallback_used: false,
}
})
} else {
None
};
Ok((symbols, has_error, diagnostic, references, alias_map))
}
static IMPORT_TYPE_ARRAY_SUFFIX_RE: std::sync::LazyLock<regex::Regex> =
std::sync::LazyLock::new(|| {
regex::Regex::new(
r#"(import\s*\(\s*['"][^'"]+['"]\s*\)\s*\.\s*[A-Za-z_$][A-Za-z0-9_$]*)(?:\s*\[\s*\])+"#,
)
.expect("SF-003 import-type-array regex is a valid pattern")
});
pub(crate) fn is_expected_typescript_import_type_array_limitation(
language: &LanguageId,
content: &[u8],
) -> bool {
if !matches!(language, LanguageId::TypeScript) {
return false;
}
let source = String::from_utf8_lossy(content);
if !IMPORT_TYPE_ARRAY_SUFFIX_RE.is_match(&source) {
return false;
}
let original_has_error = match panic::catch_unwind(|| parse_source(&source, language)) {
Ok(Ok((_, has_error, _, _, _))) => has_error,
_ => return false,
};
if !original_has_error {
return false;
}
let neutralized = IMPORT_TYPE_ARRAY_SUFFIX_RE.replace_all(&source, "${1} ");
match panic::catch_unwind(|| parse_source(&neutralized, language)) {
Ok(Ok((_, has_error, _, _, _))) => !has_error,
_ => false,
}
}
static ANGULAR_CONTROL_FLOW_OPENER_RE: std::sync::LazyLock<regex::Regex> =
std::sync::LazyLock::new(|| {
regex::Regex::new(
r"(?m)(^|[^A-Za-z0-9_$])(@(?:if|for|switch|defer|else\s+if)\s*\()([^()]*)(\))",
)
.expect("SF-004 Angular control-flow opener regex is a valid pattern")
});
pub(crate) fn is_expected_angular_template_control_flow_limitation(
language: &LanguageId,
content: &[u8],
) -> bool {
if !matches!(language, LanguageId::Html) {
return false;
}
let source = String::from_utf8_lossy(content);
if !ANGULAR_CONTROL_FLOW_OPENER_RE.is_match(&source) {
return false;
}
let original_has_error = match panic::catch_unwind(|| parse_source(&source, language)) {
Ok(Ok((_, has_error, _, _, _))) => has_error,
_ => return false,
};
if !original_has_error {
return false;
}
let neutralized =
ANGULAR_CONTROL_FLOW_OPENER_RE.replace_all(&source, |caps: ®ex::Captures| {
format!(
"{}{}{}{}",
&caps[1],
&caps[2],
caps[3].replace(['<', '>'], " "),
&caps[4],
)
});
match panic::catch_unwind(|| parse_source(&neutralized, language)) {
Ok(Ok((_, has_error, _, _, _))) => !has_error,
_ => false,
}
}
pub fn extract_symbols_for_diff(source: &str, path: &str) -> Option<Vec<(String, String)>> {
let ext = path.rsplit('.').next().unwrap_or("");
let language = LanguageId::from_extension(ext)?;
if config_extractors::is_config_language(&language) {
return None; }
let result = panic::catch_unwind(|| parse_source(source, &language));
let (symbols, ..) = match result {
Ok(Ok(output)) => output,
_ => return None,
};
let pairs: Vec<(String, String)> = symbols
.iter()
.map(|sym| {
let (start, end) = sym.byte_range;
let body = &source[start as usize..end as usize];
(sym.name.clone(), crate::hash::digest_hex(body.as_bytes()))
})
.collect();
Some(pairs)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::domain::{FileOutcome, LanguageId, SymbolKind};
#[test]
fn test_process_file_rust_extracts_function() {
let source = b"fn hello() { }";
let result = process_file("test.rs", source, LanguageId::Rust);
assert_eq!(result.outcome, FileOutcome::Processed);
assert!(!result.symbols.is_empty());
assert_eq!(result.symbols[0].name, "hello");
assert_eq!(result.symbols[0].kind, SymbolKind::Function);
}
#[test]
fn test_process_file_rust_accepts_borrowed_raw_identifier() {
let source = b"fn main() { let raw = 1; let _x = &raw; }";
let result = process_file("test.rs", source, LanguageId::Rust);
assert_eq!(result.outcome, FileOutcome::Processed);
}
#[test]
fn test_process_file_rust_preserves_raw_borrow_syntax() {
for source in [
b"fn main() { let value = 1; let _ptr = &raw const value; }" as &[u8],
b"fn main() { let mut value = 1; let _ptr = &raw mut value; }",
] {
let result = process_file("test.rs", source, LanguageId::Rust);
assert_eq!(result.outcome, FileOutcome::Processed);
}
}
#[test]
fn test_process_file_python_extracts_function() {
let source = b"def greet():\n pass";
let result = process_file("test.py", source, LanguageId::Python);
assert_eq!(result.outcome, FileOutcome::Processed);
assert!(!result.symbols.is_empty());
assert_eq!(result.symbols[0].name, "greet");
assert_eq!(result.symbols[0].kind, SymbolKind::Function);
}
#[test]
fn test_process_file_javascript_extracts_function() {
let source = b"function doStuff() { }";
let result = process_file("test.js", source, LanguageId::JavaScript);
assert_eq!(result.outcome, FileOutcome::Processed);
assert!(!result.symbols.is_empty());
assert_eq!(result.symbols[0].name, "doStuff");
assert_eq!(result.symbols[0].kind, SymbolKind::Function);
}
#[test]
fn test_process_file_typescript_extracts_interface() {
let source = b"interface Greeter { greet(): void; }";
let result = process_file("test.ts", source, LanguageId::TypeScript);
assert_eq!(result.outcome, FileOutcome::Processed);
let interface = result
.symbols
.iter()
.find(|s| s.kind == SymbolKind::Interface);
assert!(interface.is_some());
assert_eq!(interface.unwrap().name, "Greeter");
}
#[test]
fn test_sf003_class_field_import_type_array_is_expected_limitation() {
let source = b"class C { private subs: import('rxjs').Subscription[] = []; }";
let result = process_file(
"workflow-builder.component.ts",
source,
LanguageId::TypeScript,
);
assert!(
matches!(result.outcome, FileOutcome::PartialParse { .. }),
"expected grammar still reports a partial parse for the limitation case"
);
assert!(
is_expected_typescript_import_type_array_limitation(&LanguageId::TypeScript, source),
"class-field import-type-array must be recognized as an expected grammar limitation"
);
assert!(
result.symbols.iter().any(|s| s.name == "C"),
"class symbol C must still be extracted despite the partial parse"
);
}
#[test]
fn test_sf003_type_alias_import_type_array_is_expected_limitation() {
let source = b"type S = import('rxjs').Subscription[];";
let result = process_file("types.ts", source, LanguageId::TypeScript);
assert!(
matches!(result.outcome, FileOutcome::PartialParse { .. }),
"type-alias import-type-array still reports a partial parse"
);
assert!(
is_expected_typescript_import_type_array_limitation(&LanguageId::TypeScript, source),
"type-alias import-type-array must be recognized as an expected grammar limitation"
);
}
#[test]
fn test_sf003_multidim_import_type_array_is_expected_limitation() {
let source = b"type S = import('rxjs').Subscription[][];";
assert!(
is_expected_typescript_import_type_array_limitation(&LanguageId::TypeScript, source),
"multi-dimensional import-type-array must be recognized as an expected limitation"
);
}
#[test]
fn test_sf003_negative_control_genuinely_broken_array_stays_partial() {
let source = b"class C { private subs: import('rxjs').Subscription[] = [ ; foo bar baz }";
let result = process_file("broken.ts", source, LanguageId::TypeScript);
assert!(
matches!(result.outcome, FileOutcome::PartialParse { .. }),
"the genuinely broken file is a partial parse"
);
assert!(
!is_expected_typescript_import_type_array_limitation(&LanguageId::TypeScript, source),
"a genuinely broken file sharing the import-type-array prefix MUST stay partial"
);
}
#[test]
fn test_sf003_negative_control_identifier_glue_stays_partial() {
let source = b"type S = import('x').Sub[]scription;";
let result = process_file("glue-type-alias.ts", source, LanguageId::TypeScript);
assert!(
matches!(result.outcome, FileOutcome::PartialParse { .. }),
"the genuinely broken identifier-glue file is a partial parse"
);
assert!(
!is_expected_typescript_import_type_array_limitation(&LanguageId::TypeScript, source),
"deleting `[]` would glue `Sub[]scription` into a valid identifier; the \
token-preserving neutralization must keep this broken file partial"
);
}
#[test]
fn test_sf003_negative_control_identifier_glue_class_field_stays_partial() {
let source = b"class C { private a: import('rxjs').Sub[]scription = x; }";
let result = process_file("glue-class-field.ts", source, LanguageId::TypeScript);
assert!(
matches!(result.outcome, FileOutcome::PartialParse { .. }),
"the genuinely broken class-field identifier-glue file is a partial parse"
);
assert!(
!is_expected_typescript_import_type_array_limitation(&LanguageId::TypeScript, source),
"a class-field import-type member glued to a trailing identifier fragment \
must stay partial under the token-preserving neutralization"
);
}
#[test]
fn test_sf003_negative_control_unrelated_syntax_error_stays_partial() {
let source = b"class C { private x: = ; }";
assert!(
!is_expected_typescript_import_type_array_limitation(&LanguageId::TypeScript, source),
"an unrelated syntax error must not be excused as an import-type-array limitation"
);
}
#[test]
fn test_sf003_negative_control_valid_array_plus_real_error_stays_partial() {
let source = b"type S = import('rxjs').Subscription[]; class C { private x: = ; }";
assert!(
!is_expected_typescript_import_type_array_limitation(&LanguageId::TypeScript, source),
"a real error elsewhere must keep the file partial even when an import-type-array is present"
);
}
#[test]
fn test_sf003_detector_is_typescript_gated() {
let source = b"type S = import('rxjs').Subscription[];";
assert!(
!is_expected_typescript_import_type_array_limitation(&LanguageId::JavaScript, source),
"the import-type-array limitation detector must be TypeScript-gated"
);
}
#[test]
fn test_sf003_detector_false_on_clean_typescript() {
let source = b"type S = import('rxjs').Subscription;";
assert!(
!is_expected_typescript_import_type_array_limitation(&LanguageId::TypeScript, source),
"clean scalar import-type must not be classified as a limitation"
);
}
#[test]
fn test_process_file_go_extracts_function() {
let source = b"package main\nfunc main() { }";
let result = process_file("test.go", source, LanguageId::Go);
assert_eq!(result.outcome, FileOutcome::Processed);
assert!(!result.symbols.is_empty());
assert_eq!(result.symbols[0].name, "main");
assert_eq!(result.symbols[0].kind, SymbolKind::Function);
}
#[test]
fn test_process_file_partial_parse() {
let source = b"fn broken( { }";
let result = process_file("bad.rs", source, LanguageId::Rust);
assert!(matches!(result.outcome, FileOutcome::PartialParse { .. }));
}
#[test]
fn test_process_file_partial_parse_has_diagnostic() {
let source = b"fn broken( { }";
let result = process_file("bad.rs", source, LanguageId::Rust);
assert!(matches!(result.outcome, FileOutcome::PartialParse { .. }));
let diag = result
.parse_diagnostic
.expect("should have a diagnostic for partial parse");
assert_eq!(diag.parser, "tree-sitter");
assert!(diag.line.is_some(), "diagnostic should have a line number");
assert!(
diag.column.is_some(),
"diagnostic should have a column number"
);
assert!(
diag.byte_span.is_some(),
"diagnostic should have a byte span"
);
assert!(
diag.message.contains("syntax"),
"message should describe the error"
);
}
#[test]
fn test_process_file_partial_parse_diagnostic_pins_location() {
let source = b"fn foo() {}\nfn bar() {}\nfn broken( { }";
let result = process_file("multi.rs", source, LanguageId::Rust);
assert!(
matches!(result.outcome, FileOutcome::PartialParse { .. }),
"source with a line-3 syntax error should be partial-parsed; got {:?}",
result.outcome
);
let diag = result
.parse_diagnostic
.expect("partial parse must attach a ParseDiagnostic");
assert_eq!(diag.parser, "tree-sitter");
assert!(
!diag.fallback_used,
"tree-sitter parses must not set fallback_used; that flag is reserved \
for config extractors that recover via a secondary parser"
);
let line = diag.line.expect("diagnostic must carry a line number");
assert!(
line >= 3,
"error is on line 3 of the source; diagnostic reported line {line}"
);
let column = diag.column.expect("diagnostic must carry a column number");
assert!(column >= 1, "columns are 1-based; got {column}");
let (span_start, span_end) = diag
.byte_span
.expect("diagnostic must carry a byte span for downstream editors");
assert!(
span_start <= span_end,
"byte_span must be ordered; got {span_start}..{span_end}"
);
assert!(
(span_end as usize) <= source.len(),
"byte_span must fit inside source (len {}); got end {span_end}",
source.len()
);
assert!(
span_start >= 24,
"byte_span should point at line 3 content (starts at byte 24); got {span_start}"
);
let loc = diag
.location_display()
.expect("location_display must render when both line and column are present");
assert!(
loc.contains(&format!("line {line}")),
"location_display must include the line; got {loc}"
);
assert!(
loc.contains(&format!("column {column}")),
"location_display must include the column; got {loc}"
);
let summary = diag.summary();
assert!(
summary.contains("tree-sitter:"),
"summary should prefix with parser name; got {summary}"
);
assert!(
summary.contains(&format!("line {line}")),
"summary should include location for downstream display; got {summary}"
);
}
#[test]
fn test_parse_source_reports_deepest_actionable_nested_error_node() {
let source =
"def compute():\n value = outer(inner(1 + 2, tail)\n other = {'a': {'b': 1}}\n";
let (_symbols, has_error, diagnostic, _references, _aliases) =
parse_source(source, &LanguageId::Python).expect("python parse should complete");
assert!(has_error, "fixture must contain a tree-sitter parse error");
let diag = diagnostic.expect("partial parse must attach a diagnostic");
assert_eq!(diag.parser, "tree-sitter");
assert_eq!(diag.line, Some(2), "unexpected diagnostic: {diag:?}");
assert_eq!(diag.column, Some(19), "unexpected diagnostic: {diag:?}");
assert_eq!(
diag.byte_span,
Some((33, 51)),
"diagnostic must pin the nested call error, not the outer assignment; got {diag:?}"
);
assert!(
diag.message
.contains("syntax error near `inner(1 + 2, tail)`"),
"diagnostic should include the nested call context; got {diag:?}"
);
}
#[test]
fn test_parse_source_reports_actionable_missing_node_context() {
let source = "fn compute() {\n let value = outer(inner(1 + ), tail);\n}\n";
let (_symbols, has_error, diagnostic, _references, _aliases) =
parse_source(source, &LanguageId::Rust).expect("rust parse should complete");
assert!(has_error, "fixture must contain a tree-sitter parse error");
let diag = diagnostic.expect("partial parse must attach a diagnostic");
assert_eq!(diag.parser, "tree-sitter");
assert_eq!(diag.line, Some(2), "unexpected diagnostic: {diag:?}");
assert_eq!(diag.column, Some(32), "unexpected diagnostic: {diag:?}");
assert_eq!(
diag.byte_span,
Some((46, 46)),
"diagnostic must pin the zero-width missing node; got {diag:?}"
);
assert!(
diag.message.contains("syntax missing identifier"),
"diagnostic should name the missing node kind; got {diag:?}"
);
assert!(
diag.message.contains("), tail"),
"diagnostic should include source context after the missing node; got {diag:?}"
);
}
#[test]
fn test_process_file_computes_content_hash() {
let source = b"fn foo() {}";
let result = process_file("hash_test.rs", source, LanguageId::Rust);
assert!(!result.content_hash.is_empty());
assert_eq!(result.content_hash, digest_hex(source));
}
#[test]
fn test_process_file_byte_len() {
let source = b"fn bar() {}";
let result = process_file("len.rs", source, LanguageId::Rust);
assert_eq!(result.byte_len, source.len() as u64);
}
#[test]
fn test_process_file_preserves_relative_path() {
let result = process_file("src/lib.rs", b"fn x() {}", LanguageId::Rust);
assert_eq!(result.relative_path, "src/lib.rs");
}
#[test]
fn test_process_file_never_panics_on_adversarial_input() {
let cases: &[(&[u8], &str, LanguageId)] = &[
(b"\xff\xfe\x00\x01", "binary.rs", LanguageId::Rust),
(b"", "empty.py", LanguageId::Python),
(&[0u8; 10000], "zeros.js", LanguageId::JavaScript),
(b"\n\n\n\n\n", "newlines.ts", LanguageId::TypeScript),
("\u{200b}\u{200b}".as_bytes(), "zwsp.go", LanguageId::Go),
(
b"\0\0\0fn main() {}\0\0",
"null_padded.rs",
LanguageId::Rust,
),
];
for &(source, path, ref lang) in cases {
let result = process_file(path, source, lang.clone());
assert_eq!(result.relative_path, path);
assert_eq!(result.byte_len, source.len() as u64);
assert!(!result.content_hash.is_empty());
}
}
#[test]
fn test_process_file_ruby_extracts_method() {
let source = b"def hello\n puts 'hi'\nend";
let result = process_file("app.rb", source, LanguageId::Ruby);
assert_eq!(result.outcome, FileOutcome::Processed);
assert!(
!result.symbols.is_empty(),
"should have symbols for Ruby source"
);
}
#[test]
fn test_process_file_is_idempotent_across_re_parses() {
let rust_src: &[u8] = b"use std::collections::HashMap;\n\
use std::fmt::Display;\n\
\n\
pub struct Cache<K, V> {\n\
store: HashMap<K, V>,\n\
}\n\
\n\
impl<K: std::hash::Hash + Eq, V> Cache<K, V> {\n\
pub fn new() -> Self { Self { store: HashMap::new() } }\n\
pub fn insert(&mut self, k: K, v: V) -> Option<V> { self.store.insert(k, v) }\n\
}\n\
\n\
pub fn make() -> Cache<String, u32> { Cache::new() }\n";
let python_src: &[u8] = b"import os\n\
import sys\n\
\n\
class Greeter:\n\
def __init__(self, name):\n\
self.name = name\n\
\n\
def greet(self):\n\
return f\"hello {self.name}\"\n\
\n\
def main():\n\
Greeter(\"world\").greet()\n";
let ts_src: &[u8] = b"interface Greeter { greet(): string; }\n\
export class Hello implements Greeter {\n\
constructor(private name: string) {}\n\
greet() { return `hi ${this.name}`; }\n\
}\n";
let broken_src: &[u8] = b"fn broken( { }";
let cases: &[(&[u8], &str, LanguageId)] = &[
(rust_src, "cache.rs", LanguageId::Rust),
(python_src, "greet.py", LanguageId::Python),
(ts_src, "hello.ts", LanguageId::TypeScript),
(broken_src, "broken.rs", LanguageId::Rust),
];
for &(source, path, ref lang) in cases {
let first = process_file(path, source, lang.clone());
let second = process_file(path, source, lang.clone());
let third = process_file(path, source, lang.clone());
assert_eq!(
first, second,
"{path}: identical bytes must yield identical FileProcessingResult (run 1 vs 2)"
);
assert_eq!(
second, third,
"{path}: identical bytes must yield identical FileProcessingResult (run 2 vs 3)"
);
}
}
#[test]
fn test_parse_source_zero_bytes() {
let result = parse_source("", &LanguageId::Rust)
.expect("parse_source must handle empty input without error");
let (symbols, _has_error, _diagnostic, references, alias_map) = result;
assert!(symbols.is_empty(), "empty source has no symbols");
assert!(references.is_empty(), "empty source has no references");
assert!(alias_map.is_empty(), "empty source has no aliases");
}
#[test]
fn test_parse_source_null_bytes_only() {
let source: String = "\0".repeat(4096);
let result = parse_source(&source, &LanguageId::Rust)
.expect("parse_source must handle null-byte input without error");
let (_symbols, _has_error, _diagnostic, _references, _aliases) = result;
}
#[test]
fn test_parse_source_wide_multibyte_error_region_no_panic() {
let source: String = "€".repeat(100);
parse_source(&source, &LanguageId::Rust)
.expect("parse_source must not panic on wide multi-byte error region");
}
#[test]
fn test_parse_source_mixed_multibyte_error_boundary_no_panic() {
let mut source = String::new();
source.push_str("struct S { ");
for _ in 0..14 {
source.push('€');
}
source.push(' ');
parse_source(&source, &LanguageId::Rust)
.expect("parse_source must not panic when error snippet spans multi-byte chars");
}
#[test]
fn test_process_file_deeply_nested_expression_no_stack_blow() {
const DEPTH: usize = 10_000;
let handle = std::thread::Builder::new()
.stack_size(16 * 1024 * 1024)
.spawn(|| {
let mut source = String::with_capacity(DEPTH * 2 + 32);
source.push_str("fn f() -> i32 { ");
for _ in 0..DEPTH {
source.push('(');
}
source.push('1');
for _ in 0..DEPTH {
source.push(')');
}
source.push_str(" }");
let result = process_file("deep.rs", source.as_bytes(), LanguageId::Rust);
assert_eq!(result.byte_len, source.len() as u64);
assert!(!result.content_hash.is_empty());
})
.expect("spawn stack-blow probe thread");
handle.join().expect("deep-nesting probe must not panic");
}
#[test]
fn test_process_file_deeply_nested_expression_on_default_stack_no_panic() {
const DEPTH: usize = 10_000;
let mut source = String::with_capacity(DEPTH * 2 + 32);
source.push_str("fn f() -> i32 { ");
for _ in 0..DEPTH {
source.push('(');
}
source.push('1');
for _ in 0..DEPTH {
source.push(')');
}
source.push_str(" }");
let result = process_file("deep_default.rs", source.as_bytes(), LanguageId::Rust);
assert_eq!(result.byte_len, source.len() as u64);
assert!(!result.content_hash.is_empty());
}
}