use super::Index;
use crate::config::Config;
use crate::gitx::Repo;
use anyhow::Result;
use std::collections::{HashMap, HashSet};
use std::path::Path;
const BUILTIN_EXCLUDES: &[&str] = &[
".git",
"node_modules",
"target",
"dist",
"build",
".next",
".venv",
"vendor",
"__pycache__",
".idea",
".vscode",
];
pub fn language_of(path: &str) -> &'static str {
let ext = Path::new(path)
.extension()
.and_then(|e| e.to_str())
.unwrap_or("")
.to_ascii_lowercase();
match ext.as_str() {
"ts" | "mts" | "cts" => "typescript",
"tsx" => "tsx",
"js" | "mjs" | "cjs" => "javascript",
"jsx" => "jsx",
"py" | "pyi" => "python",
"rs" => "rust",
"go" => "go",
"lua" => "lua",
"json" => "json",
"yaml" | "yml" => "yaml",
"toml" => "toml",
"md" | "markdown" => "markdown",
"sql" => "sql",
"sh" | "bash" | "zsh" => "shell",
"mly" => "memlay-record",
"" => "unknown",
_ => "text",
}
}
fn is_probably_binary(bytes: &[u8]) -> bool {
bytes.iter().take(4096).any(|b| *b == 0)
}
#[derive(Debug, Default, serde::Serialize)]
pub struct FileUpdate {
pub scanned: usize,
pub added: usize,
pub changed: usize,
pub removed: usize,
pub skipped_binary: usize,
pub skipped_oversize: usize,
}
pub fn update_files(index: &mut Index, repo: &Repo, cfg: &Config) -> Result<FileUpdate> {
let mut update = FileUpdate::default();
let now = chrono::Utc::now().to_rfc3339();
let mut existing: HashMap<String, String> = HashMap::new();
{
let mut stmt = index.conn.prepare("SELECT path, content_hash FROM files")?;
for row in stmt.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? {
let (p, h) = row?;
existing.insert(p, h);
}
}
let mut seen: HashSet<String> = HashSet::new();
let tx = index.conn.transaction()?;
{
let mut upsert = tx.prepare(
"INSERT INTO files(path, language, content_hash, size, origin, indexed_at, parse_status)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, 'pending')
ON CONFLICT(path) DO UPDATE SET
language = excluded.language,
content_hash = excluded.content_hash,
size = excluded.size,
origin = excluded.origin,
indexed_at = excluded.indexed_at,
parse_status = 'pending',
parse_error = NULL",
)?;
let mut del_fts = tx.prepare("DELETE FROM files_fts WHERE path = ?1")?;
let mut ins_fts = tx.prepare("INSERT INTO files_fts(path) VALUES (?1)")?;
for root in &cfg.index.roots {
let base = if root == "." {
repo.root.clone()
} else {
repo.root
.join(root.replace('/', std::path::MAIN_SEPARATOR_STR))
};
if !base.starts_with(&repo.root) {
continue; }
if !base.exists() {
continue;
}
let mut builder = ignore::WalkBuilder::new(&base);
builder
.hidden(false)
.follow_links(false)
.git_ignore(cfg.index.respect_gitignore)
.git_global(cfg.index.respect_gitignore)
.git_exclude(cfg.index.respect_gitignore)
.ignore(true)
.add_custom_ignore_filename(".memlayignore")
.filter_entry(|e| {
let name = e.file_name().to_string_lossy();
!BUILTIN_EXCLUDES.contains(&name.as_ref())
});
for entry in builder.build().flatten() {
let Some(ft) = entry.file_type() else {
continue;
};
if !ft.is_file() {
continue;
}
let abs = entry.path();
let rel = match abs.strip_prefix(&repo.root) {
Ok(r) => r.to_string_lossy().replace('\\', "/"),
Err(_) => continue,
};
if rel.starts_with(".memlay/records/") {
continue; }
let meta = match entry.metadata() {
Ok(m) => m,
Err(_) => continue,
};
if meta.len() > cfg.index.max_file_bytes {
update.skipped_oversize += 1;
continue;
}
update.scanned += 1;
let bytes = match std::fs::read(abs) {
Ok(b) => b,
Err(_) => continue,
};
if is_probably_binary(&bytes) {
update.skipped_binary += 1;
continue;
}
let hash = blake3::hash(&bytes).to_hex().to_string();
seen.insert(rel.clone());
match existing.get(&rel) {
Some(prev) if *prev == hash => continue, Some(_) => update.changed += 1,
None => update.added += 1,
}
upsert.execute(rusqlite::params![
rel,
language_of(&rel),
hash,
meta.len() as i64,
"worktree",
now,
])?;
del_fts.execute([&rel])?;
ins_fts.execute([&rel])?;
}
}
let mut del_file = tx.prepare("DELETE FROM files WHERE path = ?1")?;
let mut del_syms = tx.prepare("DELETE FROM symbols WHERE path = ?1")?;
let mut del_sym_fts = tx.prepare("DELETE FROM symbols_fts WHERE path = ?1")?;
let mut del_edges = tx.prepare("DELETE FROM symbol_edges WHERE src_path = ?1")?;
for path in existing.keys() {
if !seen.contains(path) {
update.removed += 1;
del_file.execute([path])?;
del_fts.execute([path])?;
del_syms.execute([path])?;
del_sym_fts.execute([path])?;
del_edges.execute([path])?;
}
}
}
tx.commit()?;
Ok(update)
}
#[derive(Debug, Default, serde::Serialize)]
pub struct ParseStats {
pub parsed: usize,
pub cache_hits: usize,
pub errors: usize,
pub lexical_only: usize,
pub symbols: usize,
}
pub fn parse_pending(index: &mut Index, repo: &Repo, cfg: &Config) -> Result<ParseStats> {
let mut stats = ParseStats::default();
let cache = crate::parsers::ParseCache::new(&repo.shared_dir());
let pending: Vec<(String, String, String)> = {
let mut stmt = index.conn.prepare(
"SELECT path, language, content_hash FROM files WHERE parse_status = 'pending'",
)?;
let rows = stmt.query_map([], |r| {
Ok((
r.get::<_, String>(0)?,
r.get::<_, String>(1)?,
r.get::<_, String>(2)?,
))
})?;
rows.filter_map(|r| r.ok()).collect()
};
let tx = index.conn.transaction()?;
{
let mut set_status =
tx.prepare("UPDATE files SET parse_status = ?2, parse_error = ?3 WHERE path = ?1")?;
let mut del_syms = tx.prepare("DELETE FROM symbols WHERE path = ?1")?;
let mut del_sym_fts = tx.prepare("DELETE FROM symbols_fts WHERE path = ?1")?;
let mut del_edges = tx.prepare("DELETE FROM symbol_edges WHERE src_path = ?1")?;
let mut ins_sym = tx.prepare(
"INSERT INTO symbols(name, qualified_name, kind, path, start_line, end_line, signature, content_hash, is_test)
VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)",
)?;
let mut ins_sym_fts = tx.prepare(
"INSERT INTO symbols_fts(symbol_id, name, qualified_name, path, parts) VALUES (?1,?2,?3,?4,?5)",
)?;
let mut ins_edge = tx.prepare(
"INSERT INTO symbol_edges(src_path, edge_type, dst) VALUES (?1, 'imports', ?2)",
)?;
for (path, language, content_hash) in pending {
if !crate::parsers::is_supported(&language)
|| !cfg.index.languages.iter().any(|l| {
l == &language
|| (l == "typescript" && language == "tsx")
|| (l == "javascript" && language == "jsx")
})
{
stats.lexical_only += 1;
set_status.execute(rusqlite::params![path, "lexical", Option::<String>::None])?;
continue;
}
let extraction = if let Some(cached) = cache.get(&language, &content_hash) {
stats.cache_hits += 1;
Ok(cached)
} else {
let abs = repo
.root
.join(path.replace('/', std::path::MAIN_SEPARATOR_STR));
match std::fs::read(&abs) {
Ok(bytes) => match crate::parsers::extract(&language, &path, &bytes) {
Some(Ok(e)) => {
cache.put(&language, &content_hash, &e);
stats.parsed += 1;
Ok(e)
}
Some(Err(e)) => Err(e.to_string()),
None => {
stats.lexical_only += 1;
set_status.execute(rusqlite::params![
path,
"lexical",
Option::<String>::None
])?;
continue;
}
},
Err(e) => Err(e.to_string()),
}
};
match extraction {
Ok(e) => {
del_syms.execute([&path])?;
del_sym_fts.execute([&path])?;
del_edges.execute([&path])?;
for s in &e.symbols {
ins_sym.execute(rusqlite::params![
s.name,
s.qualified_name,
s.kind,
path,
s.start_line,
s.end_line,
s.signature,
content_hash,
s.is_test as i64,
])?;
let id = tx.last_insert_rowid();
let parts =
crate::retrieval::query::split_ident(&s.qualified_name).join(" ");
ins_sym_fts.execute(rusqlite::params![
id,
s.name,
s.qualified_name,
path,
parts
])?;
stats.symbols += 1;
}
for imp in &e.imports {
ins_edge.execute(rusqlite::params![path, imp])?;
}
set_status.execute(rusqlite::params![path, "ok", Option::<String>::None])?;
}
Err(msg) => {
stats.errors += 1;
set_status.execute(rusqlite::params![path, "error", msg])?;
}
}
}
}
tx.commit()?;
Ok(stats)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::index::Index;
fn fixture() -> (tempfile::TempDir, Repo, Config) {
let tmp = tempfile::tempdir().unwrap();
let run = |args: &[&str]| {
std::process::Command::new("git")
.args(args)
.current_dir(tmp.path())
.output()
.unwrap()
};
run(&["init", "-b", "main"]);
std::fs::write(tmp.path().join("main.rs"), "fn main() {}\n").unwrap();
std::fs::create_dir_all(tmp.path().join("node_modules/dep")).unwrap();
std::fs::write(tmp.path().join("node_modules/dep/x.js"), "ignored\n").unwrap();
std::fs::write(tmp.path().join("bin.dat"), [0u8, 1, 2, 3]).unwrap();
let repo = Repo::discover(tmp.path()).unwrap();
(tmp, repo, Config::default())
}
#[test]
fn discovers_excludes_and_increments() {
let (_tmp, repo, cfg) = fixture();
let mut index = Index::open(&repo).unwrap();
let u1 = update_files(&mut index, &repo, &cfg).unwrap();
assert_eq!(u1.added, 1, "{u1:?}"); assert_eq!(u1.skipped_binary, 1);
let u2 = update_files(&mut index, &repo, &cfg).unwrap();
assert_eq!(u2.added + u2.changed + u2.removed, 0, "{u2:?}");
std::fs::write(repo.root.join("main.rs"), "fn main() { println!(); }\n").unwrap();
let u3 = update_files(&mut index, &repo, &cfg).unwrap();
assert_eq!(u3.changed, 1, "{u3:?}");
std::fs::remove_file(repo.root.join("main.rs")).unwrap();
let u4 = update_files(&mut index, &repo, &cfg).unwrap();
assert_eq!(u4.removed, 1, "{u4:?}");
}
#[test]
fn language_detection() {
assert_eq!(language_of("a/b.ts"), "typescript");
assert_eq!(language_of("a/b.tsx"), "tsx");
assert_eq!(language_of("x.py"), "python");
assert_eq!(language_of("x.rs"), "rust");
assert_eq!(language_of("x.unknownext"), "text");
}
}