use std::collections::HashMap;
use std::fmt;
#[derive(
Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize, serde::Deserialize,
)]
pub enum LanguageId {
Rust,
Python,
JavaScript,
TypeScript,
Go,
Java,
C,
Cpp,
CSharp,
Ruby,
Php,
Swift,
Kotlin,
Dart,
Perl,
Elixir,
Json,
Toml,
Yaml,
Markdown,
Env,
Html,
Css,
Scss,
}
impl LanguageId {
pub fn from_extension(ext: &str) -> Option<Self> {
match ext {
"rs" => Some(Self::Rust),
"py" => Some(Self::Python),
"js" | "jsx" => Some(Self::JavaScript),
"ts" | "tsx" => Some(Self::TypeScript),
"go" => Some(Self::Go),
"java" => Some(Self::Java),
"c" | "h" => Some(Self::C),
"cpp" | "cxx" | "cc" | "hpp" | "hxx" | "hh" => Some(Self::Cpp),
"cs" => Some(Self::CSharp),
"rb" => Some(Self::Ruby),
"php" => Some(Self::Php),
"swift" => Some(Self::Swift),
"dart" => Some(Self::Dart),
"kt" | "kts" => Some(Self::Kotlin),
"pl" | "pm" => Some(Self::Perl),
"ex" | "exs" => Some(Self::Elixir),
"json" => Some(Self::Json),
"toml" => Some(Self::Toml),
"yaml" | "yml" => Some(Self::Yaml),
"md" => Some(Self::Markdown),
"env" => Some(Self::Env),
"html" => Some(Self::Html),
"css" => Some(Self::Css),
"scss" => Some(Self::Scss),
_ => None,
}
}
pub fn is_tsx_path(relative_path: &str) -> bool {
relative_path
.rsplit(['/', '\\'])
.next()
.and_then(|name| name.rsplit_once('.'))
.is_some_and(|(_, ext)| ext.eq_ignore_ascii_case("tsx"))
}
pub fn extensions(&self) -> &[&str] {
match self {
Self::Rust => &["rs"],
Self::Python => &["py"],
Self::JavaScript => &["js", "jsx"],
Self::TypeScript => &["ts", "tsx"],
Self::Go => &["go"],
Self::Java => &["java"],
Self::C => &["c", "h"],
Self::Cpp => &["cpp", "cxx", "cc", "hpp", "hxx", "hh"],
Self::CSharp => &["cs"],
Self::Ruby => &["rb"],
Self::Php => &["php"],
Self::Swift => &["swift"],
Self::Kotlin => &["kt", "kts"],
Self::Dart => &["dart"],
Self::Perl => &["pl", "pm"],
Self::Elixir => &["ex", "exs"],
Self::Json => &["json"],
Self::Toml => &["toml"],
Self::Yaml => &["yaml", "yml"],
Self::Markdown => &["md"],
Self::Env => &["env"],
Self::Html => &["html"],
Self::Css => &["css"],
Self::Scss => &["scss"],
}
}
pub fn support_tier(&self) -> SupportTier {
match self {
Self::Rust | Self::Python | Self::JavaScript | Self::TypeScript | Self::Go => {
SupportTier::QualityFocus
}
Self::Java
| Self::C
| Self::Cpp
| Self::CSharp
| Self::Ruby
| Self::Php
| Self::Swift
| Self::Kotlin
| Self::Dart
| Self::Perl
| Self::Elixir
| Self::Json
| Self::Toml
| Self::Yaml
| Self::Markdown
| Self::Env
| Self::Html
| Self::Css
| Self::Scss => SupportTier::Broader,
}
}
}
impl fmt::Display for LanguageId {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let name = match self {
Self::Rust => "Rust",
Self::Python => "Python",
Self::JavaScript => "JavaScript",
Self::TypeScript => "TypeScript",
Self::Go => "Go",
Self::Java => "Java",
Self::C => "C",
Self::Cpp => "C++",
Self::CSharp => "C#",
Self::Ruby => "Ruby",
Self::Php => "PHP",
Self::Swift => "Swift",
Self::Kotlin => "Kotlin",
Self::Dart => "Dart",
Self::Perl => "Perl",
Self::Elixir => "Elixir",
Self::Json => "JSON",
Self::Toml => "TOML",
Self::Yaml => "YAML",
Self::Markdown => "Markdown",
Self::Env => "Env",
Self::Html => "HTML",
Self::Css => "CSS",
Self::Scss => "SCSS",
};
write!(f, "{name}")
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum SupportTier {
QualityFocus,
Broader,
Unsupported,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
pub enum FileClass {
Code,
Text,
Binary,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
pub struct FileClassification {
pub class: FileClass,
pub is_generated: bool,
pub is_test: bool,
pub is_vendor: bool,
#[serde(default)]
pub is_config: bool,
}
impl FileClassification {
pub fn for_code_path(relative_path: &str) -> Self {
let lower = relative_path.replace('\\', "/").to_ascii_lowercase();
let segments: Vec<&str> = lower
.split('/')
.filter(|segment| !segment.is_empty())
.collect();
let basename = segments.last().copied().unwrap_or("");
let stem = basename
.rsplit_once('.')
.map(|(name, _)| name)
.unwrap_or(basename);
let is_test = segments
.iter()
.any(|segment| matches!(*segment, "tests" | "test" | "__tests__" | "spec"))
|| stem.starts_with("test_")
|| stem.ends_with("_test")
|| stem.ends_with(".test")
|| stem.ends_with("_spec")
|| stem.ends_with(".spec");
let is_vendor = segments.iter().any(|segment| {
matches!(
*segment,
"vendor"
| "third_party"
| "third-party"
| "node_modules"
| ".venv"
| "venv"
| "site-packages"
| "pods"
)
});
let is_generated = segments.iter().any(|segment| {
matches!(
*segment,
"generated" | "__generated__" | "generated-sources"
)
}) || basename.contains(".generated.")
|| basename.contains(".gen.")
|| basename.ends_with(".g.dart")
|| basename.ends_with(".pb.go")
|| basename.ends_with(".designer.cs")
|| basename.ends_with(".min.js");
let ext = basename.rsplit_once('.').map(|(_, e)| e).unwrap_or("");
let is_config = matches!(ext, "json" | "toml" | "yaml" | "yml" | "md" | "env");
Self {
class: FileClass::Code,
is_generated,
is_test,
is_vendor,
is_config,
}
}
pub const fn is_code(&self) -> bool {
matches!(self.class, FileClass::Code)
}
pub const fn is_text(&self) -> bool {
matches!(self.class, FileClass::Text)
}
pub const fn is_binary(&self) -> bool {
matches!(self.class, FileClass::Binary)
}
}
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct ParseDiagnostic {
pub parser: String,
pub message: String,
pub line: Option<u32>,
pub column: Option<u32>,
pub byte_span: Option<(u32, u32)>,
pub fallback_used: bool,
}
impl ParseDiagnostic {
pub fn location_display(&self) -> Option<String> {
match (self.line, self.column) {
(Some(line), Some(column)) => Some(format!("line {line}, column {column}")),
(Some(line), None) => Some(format!("line {line}")),
(None, Some(column)) => Some(format!("column {column}")),
(None, None) => None,
}
}
pub fn summary(&self) -> String {
let mut summary = format!("{}: {}", self.parser, self.message);
if let Some(location) = self.location_display() {
summary.push_str(&format!(" ({location})"));
}
if self.fallback_used {
summary.push_str(" [fallback symbol extraction used]");
}
summary
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct FileProcessingResult {
pub relative_path: String,
pub language: LanguageId,
pub classification: FileClassification,
pub outcome: FileOutcome,
pub parse_diagnostic: Option<ParseDiagnostic>,
pub symbols: Vec<SymbolRecord>,
pub byte_len: u64,
pub content_hash: String,
pub references: Vec<ReferenceRecord>,
pub alias_map: HashMap<String, String>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum FileOutcome {
Processed,
PartialParse { warning: String },
Failed { error: String },
}
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct SymbolRecord {
pub name: String,
pub kind: SymbolKind,
pub depth: u32,
pub sort_order: u32,
pub byte_range: (u32, u32),
pub line_range: (u32, u32),
pub doc_byte_range: Option<(u32, u32)>,
#[serde(default)]
pub item_byte_range: Option<(u32, u32)>,
}
impl SymbolRecord {
pub fn item_start(&self) -> u32 {
self.item_byte_range.map_or_else(
|| {
self.doc_byte_range
.map_or(self.byte_range.0, |(start, _)| start)
},
|(start, _)| start,
)
}
pub fn item_end(&self) -> u32 {
self.item_byte_range
.map_or(self.byte_range.1, |(_, end)| end)
}
pub fn item_range(&self) -> (u32, u32) {
(self.item_start(), self.item_end())
}
pub fn core_range(&self) -> (u32, u32) {
self.byte_range
}
pub fn effective_start(&self) -> u32 {
self.doc_byte_range
.map_or(self.byte_range.0, |(start, _)| start)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub enum SymbolKind {
Function,
Method,
Class,
Struct,
Enum,
Interface,
Module,
Constant,
Variable,
Type,
Trait,
Impl,
Other,
Key,
Section,
}
impl fmt::Display for SymbolKind {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let prefix = match self {
Self::Function => "fn",
Self::Method => "fn",
Self::Class => "class",
Self::Struct => "struct",
Self::Enum => "enum",
Self::Interface => "interface",
Self::Module => "mod",
Self::Constant => "const",
Self::Variable => "let",
Self::Type => "type",
Self::Trait => "trait",
Self::Impl => "impl",
Self::Other => "other",
Self::Key => "key",
Self::Section => "section",
};
write!(f, "{prefix}")
}
}
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct ReferenceRecord {
pub name: String,
pub qualified_name: Option<String>,
pub kind: ReferenceKind,
pub byte_range: (u32, u32),
pub line_range: (u32, u32),
pub enclosing_symbol_index: Option<u32>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub enum ReferenceKind {
Call,
Import,
TypeUsage,
MacroUse,
Implements,
ValueUse,
}
impl fmt::Display for ReferenceKind {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let s = match self {
Self::Call => "call",
Self::Import => "import",
Self::TypeUsage => "type_usage",
Self::MacroUse => "macro_use",
Self::Implements => "implements",
Self::ValueUse => "value_use",
};
write!(f, "{s}")
}
}
pub fn find_enclosing_symbol(symbols: &[SymbolRecord], ref_line: u32) -> Option<u32> {
let mut best: Option<(u32, u32)> = None; for (idx, sym) in symbols.iter().enumerate() {
let (start, end) = sym.line_range;
if ref_line >= start && ref_line <= end {
match best {
None => best = Some((start, idx as u32)),
Some((best_start, _)) if start > best_start => best = Some((start, idx as u32)),
_ => {}
}
}
}
best.map(|(_, idx)| idx)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum AdmissionTier {
Normal,
MetadataOnly,
HardSkip,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SkipReason {
SizeCeiling,
DenylistedExtension,
SizeThreshold,
BinaryContent,
DependencyLockfile,
Untracked,
}
impl std::fmt::Display for SkipReason {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
SkipReason::SizeCeiling => write!(f, ">100MB"),
SkipReason::DenylistedExtension => write!(f, "artifact"),
SkipReason::SizeThreshold => write!(f, ">1MB"),
SkipReason::BinaryContent => write!(f, "binary"),
SkipReason::DependencyLockfile => write!(f, "lockfile"),
SkipReason::Untracked => write!(f, "untracked"),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct AdmissionDecision {
pub tier: AdmissionTier,
pub reason: Option<SkipReason>,
}
impl AdmissionDecision {
pub fn normal() -> Self {
Self {
tier: AdmissionTier::Normal,
reason: None,
}
}
pub fn skip(tier: AdmissionTier, reason: SkipReason) -> Self {
Self {
tier,
reason: Some(reason),
}
}
}
#[derive(Debug, Clone)]
pub struct SkippedFile {
pub path: String,
pub size: u64,
pub extension: Option<String>,
pub decision: AdmissionDecision,
}
impl SkippedFile {
pub fn tier(&self) -> AdmissionTier {
self.decision.tier
}
pub fn reason(&self) -> Option<SkipReason> {
self.decision.reason
}
}
pub const HARD_SKIP_BYTES: u64 = 100 * 1024 * 1024;
pub const METADATA_ONLY_BYTES: u64 = 1024 * 1024;
pub const BINARY_SNIFF_BYTES: usize = 8192;
const DENYLISTED_EXTENSIONS: &[&str] = &[
"safetensors",
"ckpt",
"pt",
"onnx",
"gguf",
"pth",
"vmdk",
"iso",
"img",
"qcow2",
"tar",
"gz",
"zip",
"7z",
"rar",
"bz2",
"xz",
"zst",
"db",
"sqlite",
"sqlite3",
"mdb",
"mp3",
"mp4",
"wav",
"avi",
"mov",
"mkv",
"png",
"jpg",
"jpeg",
"gif",
"bmp",
"ico",
"woff",
"woff2",
"ttf",
"eot",
"bin",
"exe",
"dll",
"so",
"dylib",
"class",
];
pub fn is_denylisted_extension(ext: &str) -> bool {
DENYLISTED_EXTENSIONS.contains(&ext.to_lowercase().as_str())
}
const LOCKFILE_BASENAMES: &[&str] = &[
"package-lock.json",
"npm-shrinkwrap.json",
"yarn.lock",
"pnpm-lock.yaml",
"bun.lock",
"Cargo.lock",
"composer.lock",
"Gemfile.lock",
"poetry.lock",
"uv.lock",
"Pipfile.lock",
"flake.lock",
"packages.lock.json",
"go.sum",
];
pub fn is_dependency_lockfile(path: &std::path::Path) -> bool {
path.file_name()
.and_then(|n| n.to_str())
.is_some_and(|name| LOCKFILE_BASENAMES.contains(&name))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_symbol_kind_display_function() {
assert_eq!(SymbolKind::Function.to_string(), "fn");
}
#[test]
fn test_symbol_kind_display_method() {
assert_eq!(SymbolKind::Method.to_string(), "fn");
}
#[test]
fn test_symbol_kind_display_class() {
assert_eq!(SymbolKind::Class.to_string(), "class");
}
#[test]
fn test_symbol_kind_display_struct() {
assert_eq!(SymbolKind::Struct.to_string(), "struct");
}
#[test]
fn test_symbol_kind_display_enum() {
assert_eq!(SymbolKind::Enum.to_string(), "enum");
}
#[test]
fn test_symbol_kind_display_interface() {
assert_eq!(SymbolKind::Interface.to_string(), "interface");
}
#[test]
fn test_symbol_kind_display_module() {
assert_eq!(SymbolKind::Module.to_string(), "mod");
}
#[test]
fn test_symbol_kind_display_constant() {
assert_eq!(SymbolKind::Constant.to_string(), "const");
}
#[test]
fn test_symbol_kind_display_variable() {
assert_eq!(SymbolKind::Variable.to_string(), "let");
}
#[test]
fn test_symbol_kind_display_type() {
assert_eq!(SymbolKind::Type.to_string(), "type");
}
#[test]
fn test_symbol_kind_display_trait() {
assert_eq!(SymbolKind::Trait.to_string(), "trait");
}
#[test]
fn test_symbol_kind_display_impl() {
assert_eq!(SymbolKind::Impl.to_string(), "impl");
}
#[test]
fn test_symbol_kind_display_other() {
assert_eq!(SymbolKind::Other.to_string(), "other");
}
#[test]
fn test_reference_kind_all_variants_constructible() {
let _call = ReferenceKind::Call;
let _import = ReferenceKind::Import;
let _type_usage = ReferenceKind::TypeUsage;
let _macro_use = ReferenceKind::MacroUse;
}
#[test]
fn test_reference_kind_display_call() {
assert_eq!(ReferenceKind::Call.to_string(), "call");
}
#[test]
fn test_reference_kind_display_import() {
assert_eq!(ReferenceKind::Import.to_string(), "import");
}
#[test]
fn test_reference_kind_display_type_usage() {
assert_eq!(ReferenceKind::TypeUsage.to_string(), "type_usage");
}
#[test]
fn test_reference_kind_display_macro_use() {
assert_eq!(ReferenceKind::MacroUse.to_string(), "macro_use");
}
#[test]
fn test_reference_record_construction_with_all_fields() {
let r = ReferenceRecord {
name: "foo".to_string(),
qualified_name: Some("Bar::foo".to_string()),
kind: ReferenceKind::Call,
byte_range: (10, 20),
line_range: (1, 1),
enclosing_symbol_index: Some(0),
};
assert_eq!(r.name, "foo");
assert_eq!(r.qualified_name.as_deref(), Some("Bar::foo"));
assert_eq!(r.kind, ReferenceKind::Call);
assert_eq!(r.byte_range, (10, 20));
assert_eq!(r.line_range, (1, 1));
assert_eq!(r.enclosing_symbol_index, Some(0));
}
#[test]
fn test_reference_record_without_optional_fields() {
let r = ReferenceRecord {
name: "baz".to_string(),
qualified_name: None,
kind: ReferenceKind::Import,
byte_range: (0, 5),
line_range: (0, 0),
enclosing_symbol_index: None,
};
assert!(r.qualified_name.is_none());
assert!(r.enclosing_symbol_index.is_none());
}
#[test]
fn test_file_processing_result_backward_compat_with_empty_refs() {
use std::collections::HashMap;
let result = FileProcessingResult {
relative_path: "test.rs".to_string(),
language: LanguageId::Rust,
classification: FileClassification::for_code_path("test.rs"),
outcome: FileOutcome::Processed,
parse_diagnostic: None,
symbols: vec![],
byte_len: 0,
content_hash: "abc".to_string(),
references: vec![],
alias_map: HashMap::new(),
};
assert!(result.references.is_empty());
assert!(result.alias_map.is_empty());
}
#[test]
fn test_file_classification_for_code_path_marks_code_only_by_default() {
let classification = FileClassification::for_code_path("src/lib.rs");
assert!(classification.is_code());
assert!(!classification.is_text());
assert!(!classification.is_binary());
assert!(!classification.is_generated);
assert!(!classification.is_test);
assert!(!classification.is_vendor);
}
#[test]
fn test_file_classification_for_code_path_marks_noise_tags_from_path() {
let generated = FileClassification::for_code_path("src/generated/client.pb.go");
assert!(generated.is_generated);
let test_file = FileClassification::for_code_path("tests/parser_spec.rs");
assert!(test_file.is_test);
let vendor_file = FileClassification::for_code_path("node_modules/pkg/index.js");
assert!(vendor_file.is_vendor);
}
#[test]
fn test_find_enclosing_symbol_innermost_for_nested() {
let symbols = vec![
SymbolRecord {
name: "outer".to_string(),
kind: SymbolKind::Function,
depth: 0,
sort_order: 0,
byte_range: (0, 100),
line_range: (0, 10),
doc_byte_range: None,
item_byte_range: None,
},
SymbolRecord {
name: "inner".to_string(),
kind: SymbolKind::Function,
depth: 1,
sort_order: 1,
byte_range: (30, 60),
line_range: (3, 6),
doc_byte_range: None,
item_byte_range: None,
},
];
let idx = find_enclosing_symbol(&symbols, 4);
assert_eq!(idx, Some(1), "should return innermost enclosing symbol");
}
#[test]
fn test_find_enclosing_symbol_none_at_module_level() {
let symbols = vec![SymbolRecord {
name: "foo".to_string(),
kind: SymbolKind::Function,
depth: 0,
sort_order: 0,
byte_range: (50, 100),
line_range: (5, 10),
doc_byte_range: None,
item_byte_range: None,
}];
let idx = find_enclosing_symbol(&symbols, 0);
assert_eq!(idx, None, "should return None when not inside any symbol");
}
#[test]
fn test_symbol_record_item_helpers_fall_back_to_doc_range() {
let symbol = SymbolRecord {
name: "target".to_string(),
kind: SymbolKind::Function,
depth: 0,
sort_order: 0,
byte_range: (20, 40),
line_range: (1, 3),
doc_byte_range: Some((10, 20)),
item_byte_range: None,
};
assert_eq!(symbol.item_start(), 10);
assert_eq!(symbol.item_end(), 40);
assert_eq!(symbol.item_range(), (10, 40));
assert_eq!(symbol.core_range(), (20, 40));
}
#[test]
fn test_symbol_record_item_helpers_prefer_item_range() {
let symbol = SymbolRecord {
name: "target".to_string(),
kind: SymbolKind::Function,
depth: 0,
sort_order: 0,
byte_range: (20, 40),
line_range: (1, 3),
doc_byte_range: Some((10, 20)),
item_byte_range: Some((5, 44)),
};
assert_eq!(symbol.item_start(), 5);
assert_eq!(symbol.item_end(), 44);
assert_eq!(symbol.item_range(), (5, 44));
assert_eq!(symbol.core_range(), (20, 40));
}
#[test]
fn test_admission_tier_variants() {
let t1 = AdmissionTier::Normal;
let t2 = AdmissionTier::MetadataOnly;
let t3 = AdmissionTier::HardSkip;
assert_ne!(t1, t2);
assert_ne!(t2, t3);
assert_ne!(t1, t3);
}
#[test]
fn test_extension_is_denylisted() {
assert!(is_denylisted_extension("safetensors"));
assert!(is_denylisted_extension("ckpt"));
assert!(is_denylisted_extension("zip"));
assert!(is_denylisted_extension("mp4"));
assert!(is_denylisted_extension("woff2"));
assert!(is_denylisted_extension("png"));
assert!(is_denylisted_extension("bin"));
}
#[test]
fn test_extension_not_denylisted() {
assert!(!is_denylisted_extension("rs"));
assert!(!is_denylisted_extension("ts"));
assert!(!is_denylisted_extension("json"));
assert!(!is_denylisted_extension("svg")); assert!(!is_denylisted_extension("md"));
assert!(!is_denylisted_extension("toml"));
}
#[test]
fn test_new_executable_extensions_denylisted() {
assert!(is_denylisted_extension("exe"));
assert!(is_denylisted_extension("dll"));
assert!(is_denylisted_extension("so"));
assert!(is_denylisted_extension("dylib"));
assert!(is_denylisted_extension("class"));
}
#[test]
fn test_denylist_case_insensitive() {
assert!(is_denylisted_extension("DLL"));
assert!(is_denylisted_extension("So"));
assert!(is_denylisted_extension("EXE"));
assert!(is_denylisted_extension("Dylib"));
assert!(is_denylisted_extension("CLASS"));
}
#[test]
fn test_skipped_file_creation() {
let decision =
AdmissionDecision::skip(AdmissionTier::MetadataOnly, SkipReason::DenylistedExtension);
let sf = SkippedFile {
path: "model.safetensors".into(),
size: 4_200_000_000,
extension: Some("safetensors".into()),
decision,
};
assert_eq!(sf.tier(), AdmissionTier::MetadataOnly);
assert_eq!(sf.reason(), Some(SkipReason::DenylistedExtension));
}
}