use std::collections::{BTreeMap, HashSet};
use crate::{
domain::{
CodeFileDiagnostic, CodeImportRecord, CodeParseStatus, CodeRouteRecord,
RepositoryCodeFileRecord, RepositoryCodeRange, RepositoryCodeReferenceRecord,
RepositoryCodeSymbolRecord, SymbolRole,
},
project::KNOWLEDGE_MAP_RELATIVE_PATH,
};
use tree_sitter::Node;
use super::{
chunks::{add_file_chunk, chunks_for_symbols},
dependencies::{collect_dependencies, dependency_manifest_is_facts_only},
imports::collect_imports,
manual::collect_manual_nodes,
records::{records_from_captures, upsert_symbol},
recovery,
routes::{ANONYMOUS_ROUTE_HANDLER_NAME, detect_routes},
syntax::{extract_tag_captures_safely, parse_tree_safely},
text::{count_lines, validate_text_content},
};
use crate::code::{
CodeIndexError, SnapshotBuild, config_files,
feature_flags::{FeatureFlagFileInput, extract_feature_flags},
generated_detection,
languages::{LanguageSpec, detect_language},
stable_content_hash, stable_id,
};
pub(in crate::code) fn parse_indexed_file(
build: &mut SnapshotBuild,
path: &str,
bytes: &[u8],
) -> Result<(), CodeIndexError> {
let blob_hash = stable_content_hash(bytes);
let file_id = stable_id(
"file",
[&build.repository_id, &build.source_scope, path, &blob_hash],
);
let language = detect_language(path);
let line_count = count_lines(bytes);
let is_generated = generated_detection::is_generated_file(path, bytes);
let (parse_status, degraded_reason, content) = validate_text_content(path, bytes, language)?;
let Some(content) = content else {
record_file_status(
build,
FileStatusInput {
path,
file_id: &file_id,
language_id: language.map_or("unknown", |spec| spec.id),
blob_hash: &blob_hash,
byte_len: bytes.len(),
line_count,
parse_status,
is_generated,
degraded_reason,
},
);
return Ok(());
};
let Some(language) = language else {
record_file_status(
build,
FileStatusInput {
path,
file_id: &file_id,
language_id: "unknown",
blob_hash: &blob_hash,
byte_len: bytes.len(),
line_count,
parse_status,
is_generated,
degraded_reason,
},
);
if dependency_manifest_is_facts_only(path) {
record_dependencies(build, path, &file_id, &content)?;
return Ok(());
}
add_file_chunk(build, path, &file_id, "unknown", &content)?;
record_dependencies(build, path, &file_id, &content)?;
record_feature_flags(build, path, &file_id, "unknown", &content, None)?;
return Ok(());
};
if parse_status == CodeParseStatus::TextOnly {
record_file_status(
build,
FileStatusInput {
path,
file_id: &file_id,
language_id: language.id,
blob_hash: &blob_hash,
byte_len: bytes.len(),
line_count,
parse_status,
is_generated,
degraded_reason,
},
);
record_text_only_topic_symbols(build, path, &file_id, language.id, bytes)?;
add_file_chunk(build, path, &file_id, language.id, &content)?;
record_dependencies(build, path, &file_id, &content)?;
record_feature_flags(build, path, &file_id, language.id, &content, None)?;
record_routes(build, path, &file_id, language.id, &content);
return Ok(());
}
parse_syntax_file(
build,
SyntaxFileInput {
path,
file_id: &file_id,
language,
blob_hash: &blob_hash,
byte_len: bytes.len(),
line_count,
is_generated,
content: &content,
},
)
}
struct FileStatusInput<'a> {
path: &'a str,
file_id: &'a str,
language_id: &'a str,
blob_hash: &'a str,
byte_len: usize,
line_count: usize,
parse_status: CodeParseStatus,
is_generated: bool,
degraded_reason: Option<String>,
}
fn record_file_status(build: &mut SnapshotBuild, input: FileStatusInput<'_>) {
build.files.push(RepositoryCodeFileRecord {
repository_id: build.repository_id.clone(),
source_scope: build.source_scope.clone(),
file_id: input.file_id.to_owned(),
path: input.path.to_owned(),
language_id: input.language_id.to_owned(),
blob_hash: input.blob_hash.to_owned(),
byte_len: input.byte_len,
line_count: input.line_count,
parse_status: input.parse_status,
is_generated: input.is_generated,
degraded_reason: input.degraded_reason.clone(),
});
if let Some(message) = input.degraded_reason {
build.diagnostics.push(CodeFileDiagnostic {
repository_id: build.repository_id.clone(),
source_scope: build.source_scope.clone(),
path: input.path.to_owned(),
parse_status: input.parse_status,
message,
});
}
}
fn record_text_only_topic_symbols(
build: &mut SnapshotBuild,
path: &str,
file_id: &str,
language_id: &str,
bytes: &[u8],
) -> Result<(), CodeIndexError> {
if !text_only_topic_source(path, language_id) {
return Ok(());
}
let context = FileParseContext {
build,
path,
file_id,
language_id,
content: "",
};
let mut output = FileParseOutput::new();
match language_id {
"markdown" => record_text_only_markdown_headings(&context, &mut output, bytes)?,
"yaml" if path == KNOWLEDGE_MAP_RELATIVE_PATH => {
record_text_only_knowledge_map_topics(&context, &mut output, bytes)?;
}
_ => {}
}
build.symbols.extend(output.symbols);
Ok(())
}
fn text_only_topic_source(path: &str, language_id: &str) -> bool {
language_id == "markdown" || (language_id == "yaml" && path == KNOWLEDGE_MAP_RELATIVE_PATH)
}
fn record_text_only_markdown_headings(
context: &FileParseContext<'_>,
output: &mut FileParseOutput,
bytes: &[u8],
) -> Result<(), CodeIndexError> {
let mut fence = None;
scan_text_only_lines(bytes, |line| {
if let Some(active) = fence {
if markdown_structural_line(line.text)
.is_some_and(|trimmed| closes_markdown_fence(trimmed, active))
{
fence = None;
}
return Ok(());
}
let Some(trimmed) = markdown_structural_line(line.text) else {
return Ok(());
};
if let Some(marker) = markdown_fence_marker(trimmed) {
fence = Some(marker);
return Ok(());
}
let level = trimmed
.chars()
.take_while(|character| *character == '#')
.count();
if (1..=6).contains(&level) && trimmed.as_bytes().get(level) == Some(&b' ') {
record_text_only_symbol(context, output, trimmed[level..].trim(), "heading", &line)?;
}
Ok(())
})
}
fn record_text_only_knowledge_map_topics(
context: &FileParseContext<'_>,
output: &mut FileParseOutput,
bytes: &[u8],
) -> Result<(), CodeIndexError> {
let mut in_topics = false;
let mut topic_list_indent = None;
let mut topic_item_indent = None;
scan_text_only_lines(bytes, |line| {
let code = yaml_code_prefix(line.text);
let trimmed = code.trim();
if let Some(section) = top_level_yaml_section(code) {
in_topics = section == "topics";
topic_list_indent = None;
topic_item_indent = None;
return Ok(());
}
if !in_topics || trimmed.is_empty() {
return Ok(());
}
let indent = leading_spaces(code);
if let Some(item) = trimmed.strip_prefix("- ") {
if !accept_text_only_topic_item_indent(&mut topic_list_indent, indent) {
return Ok(());
}
topic_item_indent = Some(indent);
let item = item.trim_start();
if let Some(id) = item.strip_prefix("id:") {
record_text_only_knowledge_map_topic(context, output, id, &line)?;
}
return Ok(());
}
if trimmed == "-" {
if !accept_text_only_topic_item_indent(&mut topic_list_indent, indent) {
return Ok(());
}
topic_item_indent = Some(indent);
return Ok(());
}
if topic_item_indent.is_some_and(|item_indent| indent == item_indent + 2) {
let Some(id) = trimmed.strip_prefix("id:") else {
return Ok(());
};
record_text_only_knowledge_map_topic(context, output, id, &line)?;
}
Ok(())
})
}
fn record_text_only_knowledge_map_topic(
context: &FileParseContext<'_>,
output: &mut FileParseOutput,
value: &str,
line: &TextOnlyLine<'_>,
) -> Result<(), CodeIndexError> {
let name = value.trim().trim_matches('"').trim_matches('\'');
record_text_only_symbol(context, output, name, "knowledge_map_topic", line)
}
fn accept_text_only_topic_item_indent(
topic_list_indent: &mut Option<usize>,
indent: usize,
) -> bool {
match *topic_list_indent {
Some(list_indent) => indent == list_indent,
None => {
*topic_list_indent = Some(indent);
true
}
}
}
fn record_text_only_symbol(
context: &FileParseContext<'_>,
output: &mut FileParseOutput,
name: &str,
kind: &'static str,
line: &TextOnlyLine<'_>,
) -> Result<(), CodeIndexError> {
if name.is_empty() {
return Ok(());
}
let qualified_name = format!("{}::{name}", text_only_module_path(context.path));
let symbol_snapshot_id = stable_id(
"symbol",
[
&context.build.repository_id,
&context.build.source_scope,
context.path,
&qualified_name,
&line.byte_start.to_string(),
&line.byte_end.to_string(),
],
);
let symbol = RepositoryCodeSymbolRecord {
repository_id: context.build.repository_id.clone(),
source_scope: context.build.source_scope.clone(),
symbol_snapshot_id,
canonical_symbol_id: qualified_name.clone(),
file_id: context.file_id.to_owned(),
path: context.path.to_owned(),
language_id: context.language_id.to_owned(),
name: name.to_owned(),
qualified_name,
kind: kind.to_owned(),
signature: text_only_signature(line.text, name),
doc_comment: None,
byte_range: RepositoryCodeRange::new("byte_range", line.byte_start, line.byte_end)
.map_err(|error| CodeIndexError::InvalidInput(error.to_string()))?,
line_range: RepositoryCodeRange::new("line_range", line.number, line.number)
.map_err(|error| CodeIndexError::InvalidInput(error.to_string()))?,
symbol_role: None,
};
upsert_symbol(output, symbol);
Ok(())
}
fn scan_text_only_lines(
bytes: &[u8],
mut visit: impl FnMut(TextOnlyLine<'_>) -> Result<(), CodeIndexError>,
) -> Result<(), CodeIndexError> {
let mut byte_start = 0usize;
for (index, raw_line) in bytes.split_inclusive(|byte| *byte == b'\n').enumerate() {
let without_lf = raw_line.strip_suffix(b"\n").unwrap_or(raw_line);
let text_bytes = without_lf.strip_suffix(b"\r").unwrap_or(without_lf);
let Ok(text) = std::str::from_utf8(text_bytes) else {
byte_start += raw_line.len();
continue;
};
visit(TextOnlyLine {
number: index + 1,
byte_start,
byte_end: byte_start + text_bytes.len(),
text,
})?;
byte_start += raw_line.len();
}
Ok(())
}
struct TextOnlyLine<'a> {
number: usize,
byte_start: usize,
byte_end: usize,
text: &'a str,
}
fn text_only_module_path(path: &str) -> String {
path.rsplit_once('.')
.map_or(path, |(base, _)| base)
.replace(['/', '\\'], "::")
}
fn text_only_signature(line: &str, fallback: &str) -> String {
const MAX_SIGNATURE_BYTES: usize = 512;
let trimmed = line.trim();
if trimmed.is_empty() {
return fallback.to_owned();
}
let mut signature = String::new();
for character in trimmed.chars() {
if signature.len().saturating_add(character.len_utf8()) > MAX_SIGNATURE_BYTES {
break;
}
signature.push(character);
}
if signature.is_empty() {
fallback.to_owned()
} else {
signature
}
}
fn markdown_structural_line(line: &str) -> Option<&str> {
if line.starts_with('\t') {
return None;
}
let spaces = line
.chars()
.take_while(|character| *character == ' ')
.count();
(spaces <= 3).then(|| &line[spaces..])
}
fn markdown_fence_marker(trimmed: &str) -> Option<(char, usize)> {
let marker = trimmed
.chars()
.next()
.filter(|character| matches!(*character, '`' | '~'))?;
let count = trimmed
.chars()
.take_while(|character| *character == marker)
.count();
(count >= 3).then_some((marker, count))
}
fn closes_markdown_fence(trimmed: &str, active: (char, usize)) -> bool {
let (marker, count) = active;
trimmed
.chars()
.take_while(|character| *character == marker)
.count()
>= count
}
fn leading_spaces(line: &str) -> usize {
line.chars()
.take_while(|character| *character == ' ')
.count()
}
fn yaml_code_prefix(line: &str) -> &str {
let mut in_single = false;
let mut in_double = false;
let mut escaped = false;
for (index, character) in line.char_indices() {
match character {
'\\' if in_double && !escaped => escaped = true,
'"' if !in_single && !escaped => in_double = !in_double,
'\'' if !in_double => in_single = !in_single,
'#' if !in_single && !in_double => return &line[..index],
_ => escaped = false,
}
if character != '\\' {
escaped = false;
}
}
line
}
fn top_level_yaml_section(line: &str) -> Option<&str> {
if line.starts_with(' ') || line.starts_with('\t') {
return None;
}
let key = line.trim().strip_suffix(':')?;
if key.is_empty() || key.contains(' ') {
return None;
}
Some(key)
}
pub(in crate::code::parser) struct SyntaxFileInput<'a> {
pub(in crate::code::parser) path: &'a str,
pub(in crate::code::parser) file_id: &'a str,
pub(in crate::code::parser) language: LanguageSpec,
pub(in crate::code::parser) blob_hash: &'a str,
pub(in crate::code::parser) byte_len: usize,
pub(in crate::code::parser) line_count: usize,
pub(in crate::code::parser) is_generated: bool,
pub(in crate::code::parser) content: &'a str,
}
pub(in crate::code::parser) fn parse_syntax_file(
build: &mut SnapshotBuild,
input: SyntaxFileInput<'_>,
) -> Result<(), CodeIndexError> {
let parsed = match parse_tree_safely(input.language, input.content) {
Ok(parsed) => parsed,
Err(error) => {
record_tree_sitter_failure(build, &input, "parse", &error);
record_feature_flags(
build,
input.path,
input.file_id,
input.language.id,
input.content,
None,
)?;
record_routes(
build,
input.path,
input.file_id,
input.language.id,
input.content,
);
return Ok(());
}
};
let root = parsed.root_node();
let captures = match extract_tag_captures_safely(input.language, root, input.content) {
Ok(captures) => captures,
Err(error) => {
record_tree_sitter_failure(build, &input, "query", &error);
record_feature_flags(
build,
input.path,
input.file_id,
input.language.id,
input.content,
None,
)?;
record_routes(
build,
input.path,
input.file_id,
input.language.id,
input.content,
);
return Ok(());
}
};
let context = FileParseContext {
build,
path: input.path,
file_id: input.file_id,
language_id: input.language.id,
content: input.content,
};
let mut output = FileParseOutput::new();
let (config_definitions, config_references) =
config_files::structured_facts(input.path, input.language.id, input.content);
records_from_captures(&context, captures, &mut output)?;
collect_manual_nodes(
&context,
root,
&config_definitions,
&config_references,
&mut output,
)?;
let imports = collect_imports(
build,
input.path,
input.file_id,
input.language.id,
input.content,
root,
)?;
let chunks = chunks_for_symbols(
build,
input.path,
input.file_id,
input.language.id,
input.content,
&output.symbols,
)?;
let (parse_status, degraded_reason) =
syntax_parse_status(input.language.id, root, input.content, &output, &imports);
record_file_status(
build,
FileStatusInput {
path: input.path,
file_id: input.file_id,
language_id: input.language.id,
blob_hash: input.blob_hash,
byte_len: input.byte_len,
line_count: input.line_count,
parse_status,
is_generated: input.is_generated,
degraded_reason,
},
);
build.symbols.extend(output.symbols);
build.references.extend(output.references);
build.imports.extend(imports);
record_dependencies(build, input.path, input.file_id, input.content)?;
record_feature_flags(
build,
input.path,
input.file_id,
input.language.id,
input.content,
Some(&config_definitions),
)?;
build.chunks.extend(chunks);
record_routes(
build,
input.path,
input.file_id,
input.language.id,
input.content,
);
Ok(())
}
fn syntax_parse_status(
language_id: &str,
root: Node<'_>,
content: &str,
output: &FileParseOutput,
imports: &[CodeImportRecord],
) -> (CodeParseStatus, Option<String>) {
if !root.has_error() {
return (CodeParseStatus::Parsed, None);
}
let has_structured_facts =
!(output.symbols.is_empty() && output.references.is_empty() && imports.is_empty());
if config_files::manual_parse_status(language_id, content) {
return (CodeParseStatus::Parsed, None);
}
if recovery::recoverable_c_family_parse(language_id, root, content, has_structured_facts) {
return (CodeParseStatus::Parsed, None);
}
if has_structured_facts && config_files::recoverable_parse_error(language_id, content) {
return (CodeParseStatus::Parsed, None);
}
(
CodeParseStatus::Partial,
Some("tree-sitter produced error nodes; indexed syntax facts may be partial".to_owned()),
)
}
fn record_dependencies(
build: &mut SnapshotBuild,
path: &str,
file_id: &str,
content: &str,
) -> Result<(), CodeIndexError> {
let records = collect_dependencies(build, path, file_id, content)?;
build.dependencies.extend(records);
Ok(())
}
fn record_feature_flags(
build: &mut SnapshotBuild,
path: &str,
file_id: &str,
language_id: &str,
content: &str,
config_facts: Option<&[config_files::ConfigFact]>,
) -> Result<(), CodeIndexError> {
let owned_config_facts;
let config_facts = match config_facts {
Some(config_facts) => config_facts,
None => {
owned_config_facts = config_files::structured_facts(path, language_id, content).0;
&owned_config_facts
}
};
let records = extract_feature_flags(FeatureFlagFileInput {
repository_id: &build.repository_id,
source_scope: &build.source_scope,
file_id,
path,
language_id,
content,
config_facts,
})
.map_err(|error| CodeIndexError::InvalidInput(error.to_string()))?;
build.feature_flags.extend(records);
Ok(())
}
fn record_routes(
build: &mut SnapshotBuild,
path: &str,
file_id: &str,
language_id: &str,
content: &str,
) {
let _span = tracing::debug_span!("record_routes", path, file_id, language_id).entered();
let candidates = detect_routes(language_id, content);
if candidates.is_empty() {
return;
}
tracing::debug!(route_count = candidates.len(), "detected web routes");
let symbol_index = route_handler_symbol_index(&build.symbols);
for candidate in candidates {
let route_id = stable_id(
"route",
[
&build.repository_id,
&build.source_scope,
path,
&candidate.url,
&candidate.http_method,
&candidate.handler_name,
&candidate.line.to_string(),
],
);
let line_range =
match RepositoryCodeRange::new("line_range", candidate.line, candidate.line) {
Ok(range) => range,
Err(error) => {
tracing::debug!(
path,
route_line = candidate.line,
error = %error,
"skipping route with invalid source range"
);
continue;
}
};
let route_idx = build.routes.len();
build.routes.push(CodeRouteRecord {
repository_id: build.repository_id.clone(),
source_scope: build.source_scope.clone(),
route_id,
file_id: file_id.to_owned(),
path: path.to_owned(),
language_id: language_id.to_owned(),
url: candidate.url.clone(),
http_method: candidate.http_method.clone(),
handler_name: candidate.handler_name.clone(),
handler_symbol_snapshot_id: None,
framework: candidate.framework,
line_range,
});
if candidate.handler_name != ANONYMOUS_ROUTE_HANDLER_NAME {
annotate_route_handler_symbol(
build,
&symbol_index,
RouteHandlerAnnotation {
route_idx,
path,
handler_name: &candidate.handler_name,
url: &candidate.url,
http_method: &candidate.http_method,
route_line: candidate.line,
},
);
}
}
}
fn route_handler_symbol_index(
symbols: &[RepositoryCodeSymbolRecord],
) -> BTreeMap<(String, String), Vec<usize>> {
let mut index = BTreeMap::<(String, String), Vec<usize>>::new();
for (symbol_idx, symbol) in symbols.iter().enumerate() {
index
.entry((symbol.path.clone(), symbol.name.clone()))
.or_default()
.push(symbol_idx);
}
index
}
struct RouteHandlerAnnotation<'a> {
route_idx: usize,
path: &'a str,
handler_name: &'a str,
url: &'a str,
http_method: &'a str,
route_line: usize,
}
fn annotate_route_handler_symbol(
build: &mut SnapshotBuild,
symbol_index: &BTreeMap<(String, String), Vec<usize>>,
annotation: RouteHandlerAnnotation<'_>,
) {
let route_line = annotation.route_line as u32;
let Some((symbol_name, symbol_indices)) =
route_handler_symbol_candidates(symbol_index, annotation.path, annotation.handler_name)
else {
tracing::debug!(
path = annotation.path,
handler_name = annotation.handler_name,
"route handler symbol was not found"
);
return;
};
let symbol_idx = symbol_indices
.iter()
.copied()
.filter(|idx| {
build.symbols[*idx].path == annotation.path && build.symbols[*idx].name == symbol_name
})
.min_by_key(|idx| build.symbols[*idx].line_range.start.abs_diff(route_line));
if let Some(symbol_idx) = symbol_idx {
let sym = &mut build.symbols[symbol_idx];
let symbol_snapshot_id = sym.symbol_snapshot_id.clone();
let url = annotation.url.to_owned();
let http_method = annotation.http_method.to_owned();
if let Some(role) = &mut sym.symbol_role {
role.merge_route_handler(url, http_method);
} else {
sym.symbol_role = Some(SymbolRole::RouteHandler { url, http_method });
}
if let Some(route) = build.routes.get_mut(annotation.route_idx) {
route.handler_symbol_snapshot_id = Some(symbol_snapshot_id);
}
} else {
tracing::debug!(
path = annotation.path,
handler_name = annotation.handler_name,
route_line,
"route handler symbol was not linked"
);
}
}
fn route_handler_symbol_candidates<'a>(
symbol_index: &'a BTreeMap<(String, String), Vec<usize>>,
path: &str,
handler_name: &str,
) -> Option<(String, &'a Vec<usize>)> {
if let Some(symbol_indices) = symbol_index.get(&(path.to_owned(), handler_name.to_owned())) {
return Some((handler_name.to_owned(), symbol_indices));
}
let leaf_name = handler_name.rsplit('.').next()?;
if leaf_name == handler_name {
return None;
}
symbol_index
.get(&(path.to_owned(), leaf_name.to_owned()))
.map(|symbol_indices| (leaf_name.to_owned(), symbol_indices))
}
fn record_tree_sitter_failure(
build: &mut SnapshotBuild,
input: &SyntaxFileInput<'_>,
stage: &str,
error: &CodeIndexError,
) {
record_file_status(
build,
FileStatusInput {
path: input.path,
file_id: input.file_id,
language_id: input.language.id,
blob_hash: input.blob_hash,
byte_len: input.byte_len,
line_count: input.line_count,
parse_status: CodeParseStatus::Failed,
is_generated: input.is_generated,
degraded_reason: Some(tree_sitter_failure_message(stage, error)),
},
);
}
fn tree_sitter_failure_message(stage: &str, error: &CodeIndexError) -> String {
match error {
CodeIndexError::TreeSitter(message) => {
format!("tree-sitter {stage} failed: {message}")
}
_ => error.to_string(),
}
}
pub(in crate::code::parser) struct FileParseContext<'a> {
pub(in crate::code::parser) build: &'a SnapshotBuild,
pub(in crate::code::parser) path: &'a str,
pub(in crate::code::parser) file_id: &'a str,
pub(in crate::code::parser) language_id: &'a str,
pub(in crate::code::parser) content: &'a str,
}
pub(in crate::code::parser) struct FileParseOutput {
pub(in crate::code::parser) symbols: Vec<RepositoryCodeSymbolRecord>,
pub(in crate::code::parser) references: Vec<RepositoryCodeReferenceRecord>,
pub(in crate::code::parser) reference_keys: HashSet<ReferenceDedupKey>,
}
pub(in crate::code::parser) type ReferenceDedupKey = (String, String, u32, u32, u32);
impl FileParseOutput {
pub(in crate::code::parser) fn new() -> Self {
Self {
symbols: Vec::new(),
references: Vec::new(),
reference_keys: HashSet::new(),
}
}
}
#[cfg(test)]
#[path = "file_tests.rs"]
mod tests;