mod content;
mod fuzzy;
pub mod policy;
mod store;
pub mod walk;
mod watch;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicU8, AtomicUsize, Ordering};
use std::sync::mpsc::{self, Receiver, Sender};
use std::sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard};
use std::thread::JoinHandle;
use std::time::{Duration, Instant};
pub use content::{
ContentIndex, MAX_INDEX_BYTES, TRIGRAM_LEN, extract_trigrams, pack_trigram, required_trigrams,
};
pub use fuzzy::FileMatch;
use fuzzy::FuzzyEngine;
pub use store::{Entry, IndexStore};
pub use walk::{WalkEntry, WalkStats, walk_root, walk_root_limited};
use watch::{Change, ChangeKind, Command};
fn project_meta_dir(working_dir: &Path) -> PathBuf {
working_dir.join(".whycodes")
}
pub const DEFAULT_MAX_ENTRIES: usize = 200_000;
const WATCH_DEBOUNCE: Duration = Duration::from_millis(250);
const STATE_SCANNING: u8 = 0;
const STATE_READY: u8 = 1;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ScanStatus {
Scanning {
scanned: usize,
},
Ready {
total: usize,
truncated: bool,
},
}
#[derive(Debug, Clone)]
pub struct IndexOptions {
pub watch: bool,
pub max_entries: usize,
pub threads: usize,
}
impl Default for IndexOptions {
fn default() -> Self {
Self {
watch: true,
max_entries: DEFAULT_MAX_ENTRIES,
threads: 0,
}
}
}
struct RootState {
store: RwLock<IndexStore>,
fuzzy: Mutex<FuzzyEngine>,
content: RwLock<ContentIndex>,
}
struct Shared {
roots: Vec<PathBuf>,
states: Vec<RootState>,
scanned: AtomicUsize,
state: AtomicU8,
truncated: AtomicBool,
cancel: AtomicBool,
cmd_tx: Sender<Command>,
results_dirty: Arc<AtomicBool>,
threads: usize,
max_entries: usize,
watch: bool,
}
fn lock<T>(m: &Mutex<T>) -> MutexGuard<'_, T> {
m.lock().unwrap_or_else(|e| e.into_inner())
}
fn read<T>(l: &RwLock<T>) -> RwLockReadGuard<'_, T> {
l.read().unwrap_or_else(|e| e.into_inner())
}
fn write<T>(l: &RwLock<T>) -> RwLockWriteGuard<'_, T> {
l.write().unwrap_or_else(|e| e.into_inner())
}
pub struct WorkspaceIndex {
shared: Arc<Shared>,
scanner: Option<JoinHandle<()>>,
}
impl WorkspaceIndex {
pub fn start(roots: Vec<PathBuf>) -> Arc<Self> {
Self::start_with(roots, IndexOptions::default())
}
pub fn start_with(roots: Vec<PathBuf>, opts: IndexOptions) -> Arc<Self> {
let roots = sanitize_roots(roots);
let results_dirty = Arc::new(AtomicBool::new(false));
let states = (0..roots.len())
.map(|_| RootState {
store: RwLock::new(IndexStore::new()),
fuzzy: Mutex::new(FuzzyEngine::new(results_dirty.clone())),
content: RwLock::new(ContentIndex::new()),
})
.collect();
let (cmd_tx, cmd_rx) = mpsc::channel();
let threads = if opts.threads == 0 {
std::thread::available_parallelism()
.map(|n| n.get().min(8))
.unwrap_or(1)
} else {
opts.threads
};
let shared = Arc::new(Shared {
roots,
states,
scanned: AtomicUsize::new(0),
state: AtomicU8::new(STATE_SCANNING),
truncated: AtomicBool::new(false),
cancel: AtomicBool::new(false),
cmd_tx,
results_dirty,
threads,
max_entries: opts.max_entries,
watch: opts.watch,
});
let thread_shared = shared.clone();
let scanner = std::thread::Builder::new()
.name("whycodes-index".into())
.spawn(move || scanner_main(thread_shared, cmd_rx))
.ok();
Arc::new(Self { shared, scanner })
}
pub fn project_roots(working_dir: &Path) -> Vec<PathBuf> {
let mut roots = vec![working_dir.to_path_buf()];
let allow = project_meta_dir(working_dir).join("external_dirs_allowed");
if let Ok(content) = std::fs::read_to_string(&allow) {
for line in content.lines() {
let line = line.trim();
if line.is_empty() || line.starts_with('#') {
continue;
}
roots.push(PathBuf::from(line));
}
}
roots
}
pub fn roots(&self) -> &[PathBuf] {
&self.shared.roots
}
pub fn primary_root(&self) -> Option<&Path> {
self.shared.roots.first().map(PathBuf::as_path)
}
pub fn status(&self) -> ScanStatus {
if self.shared.state.load(Ordering::Acquire) == STATE_READY {
ScanStatus::Ready {
total: self.len(),
truncated: self.shared.truncated.load(Ordering::Relaxed),
}
} else {
ScanStatus::Scanning {
scanned: self.shared.scanned.load(Ordering::Relaxed),
}
}
}
pub fn is_ready(&self) -> bool {
self.shared.state.load(Ordering::Acquire) == STATE_READY
}
pub fn len(&self) -> usize {
self.shared
.states
.iter()
.map(|s| read(&s.store).len())
.sum()
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
pub fn wait_ready(&self, timeout: Duration) -> bool {
let deadline = Instant::now() + timeout;
while !self.is_ready() && Instant::now() < deadline {
std::thread::sleep(Duration::from_millis(2));
}
self.is_ready()
}
pub fn query_now(&self, pattern: &str, limit: usize) -> Vec<FileMatch> {
let pattern = pattern.trim();
if pattern.is_empty() {
return self.browse(0, "");
}
if let Some(dir) = pattern.strip_suffix('/') {
return self.browse(0, dir);
}
for state in &self.shared.states {
lock(&state.fuzzy).set_query(pattern);
}
self.read_fuzzy(limit).0
}
pub fn read_matches(&self, limit: usize) -> Vec<FileMatch> {
self.read_fuzzy(limit).0
}
pub fn take_results_dirty(&self) -> bool {
self.shared.results_dirty.swap(false, Ordering::Acquire)
}
pub fn rearm_fuzzy(&self) {
for state in &self.shared.states {
lock(&state.fuzzy).rearm();
}
}
pub fn matching(&self) -> bool {
self.shared.states.iter().any(|s| lock(&s.fuzzy).nudge())
}
fn read_fuzzy(&self, limit: usize) -> (Vec<FileMatch>, bool) {
let mut all = Vec::new();
let mut running = false;
for (i, state) in self.shared.states.iter().enumerate() {
let (mut hits, r) = lock(&state.fuzzy).read(limit);
running |= r;
for h in &mut hits {
h.root = i as u16;
}
all.extend(hits);
}
all.sort_by(|a, b| {
b.score
.cmp(&a.score)
.then_with(|| a.rel.len().cmp(&b.rel.len()))
});
all.truncate(limit);
(all, running)
}
pub fn query(&self, pattern: &str, limit: usize) -> Vec<FileMatch> {
let pattern = pattern.trim();
if pattern.is_empty() {
return self.browse(0, "");
}
if let Some(dir) = pattern.strip_suffix('/') {
return self.browse(0, dir);
}
let mut all = Vec::new();
for (i, state) in self.shared.states.iter().enumerate() {
let mut hits = lock(&state.fuzzy).query_blocking(pattern, limit);
for h in &mut hits {
h.root = i as u16;
}
all.extend(hits);
}
all.sort_by(|a, b| {
b.score
.cmp(&a.score)
.then_with(|| a.rel.len().cmp(&b.rel.len()))
});
all.truncate(limit);
all
}
pub fn browse(&self, root: u16, rel_dir: &str) -> Vec<FileMatch> {
let Some(state) = self.shared.states.get(root as usize) else {
return Vec::new();
};
read(&state.store)
.browse(root, rel_dir)
.into_iter()
.map(|e| FileMatch {
rel: e.rel.to_string(),
is_dir: e.is_dir,
score: 0,
indices: Vec::new(),
root,
})
.collect()
}
pub fn visit(&self, f: &mut dyn FnMut(&Entry) -> bool) {
if let Some(state) = self.shared.states.first() {
for e in read(&state.store).entries() {
if !f(e) {
return;
}
}
}
}
pub fn visit_content_candidates(
&self,
search_root: &Path,
pattern: &str,
case_insensitive: bool,
f: &mut dyn FnMut(&Path, &str, u64) -> bool,
) -> Option<()> {
let required = required_trigrams(pattern, case_insensitive)?;
let state = self.shared.states.first()?;
let store = read(&state.store);
let content = read(&state.content);
if !content.is_complete() {
return None;
}
let root = self.shared.roots.first()?;
let prefix = content_search_prefix(root, search_root)?;
for e in store.entries() {
if e.is_dir || !content_entry_in_scope(&prefix, &e.rel) {
continue;
}
if !content.may_match(&e.rel, &required) {
continue;
}
let rel = content_scoped_rel(&prefix, &e.rel);
if !f(&root.join(&*e.rel), &rel, e.size) {
break;
}
}
Some(())
}
pub fn entries(&self) -> Vec<Entry> {
self.shared
.states
.first()
.map(|s| read(&s.store).entries().to_vec())
.unwrap_or_default()
}
pub fn resolve(&self, m: &FileMatch) -> PathBuf {
self.shared.roots[m.root as usize].join(&m.rel)
}
pub fn rescan(&self) {
let _ = self.shared.cmd_tx.send(Command::Rescan);
}
}
impl Drop for WorkspaceIndex {
fn drop(&mut self) {
self.shared.cancel.store(true, Ordering::Relaxed);
let _ = self.shared.cmd_tx.send(Command::Shutdown);
if let Some(h) = self.scanner.take() {
let _ = h.join();
}
}
}
impl std::fmt::Debug for WorkspaceIndex {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("WorkspaceIndex")
.field("roots", &self.shared.roots)
.field("status", &self.status())
.finish()
}
}
fn content_search_prefix(primary: &Path, search_root: &Path) -> Option<String> {
let search = std::fs::canonicalize(search_root).unwrap_or_else(|_| search_root.to_path_buf());
let rel = search.strip_prefix(primary).ok()?;
let prefix = rel.to_string_lossy().replace('\\', "/");
Some(prefix.trim_matches('/').to_string())
}
fn content_entry_in_scope(prefix: &str, rel: &str) -> bool {
prefix.is_empty()
|| (rel.len() > prefix.len()
&& rel.starts_with(prefix)
&& rel.as_bytes().get(prefix.len()) == Some(&b'/'))
}
fn content_scoped_rel(prefix: &str, rel: &str) -> String {
if prefix.is_empty() {
rel.to_string()
} else {
rel[prefix.len() + 1..].to_string()
}
}
fn sanitize_roots(roots: Vec<PathBuf>) -> Vec<PathBuf> {
let mut out: Vec<PathBuf> = Vec::new();
for root in roots {
let Ok(canon) = std::fs::canonicalize(&root) else {
continue;
};
if !canon.is_dir() {
continue;
}
if out.iter().any(|r| canon.starts_with(r)) {
continue;
}
out.push(canon);
}
out
}
#[cfg(test)]
pub(crate) fn recv_should_stop(e: mpsc::RecvTimeoutError) -> bool {
matches!(e, mpsc::RecvTimeoutError::Disconnected)
}
#[derive(Debug)]
pub(crate) enum RecvAct {
Stop,
Idle,
Rescan,
Batch(Vec<Change>),
}
pub(crate) fn classify_recv(r: Result<Command, mpsc::RecvTimeoutError>) -> RecvAct {
match r {
Ok(Command::Shutdown) => RecvAct::Stop,
Ok(Command::Rescan) => RecvAct::Rescan,
Ok(Command::Batch(cs)) => RecvAct::Batch(cs),
Err(mpsc::RecvTimeoutError::Disconnected) => RecvAct::Stop,
Err(mpsc::RecvTimeoutError::Timeout) => RecvAct::Idle,
}
}
fn scanner_main(shared: Arc<Shared>, cmd_rx: Receiver<Command>) {
full_scan(&shared);
if shared.cancel.load(Ordering::Relaxed) {
return;
}
let watcher = if shared.watch {
watch::spawn(&shared.roots, shared.cmd_tx.clone())
} else {
None
};
shared.state.store(STATE_READY, Ordering::Release);
let mut pending: Vec<Change> = Vec::new();
loop {
if shared.cancel.load(Ordering::Relaxed) {
break;
}
match classify_recv(cmd_rx.recv_timeout(WATCH_DEBOUNCE)) {
RecvAct::Stop => break,
RecvAct::Idle => {}
RecvAct::Rescan => {
pending.clear();
full_scan(&shared);
if !shared.cancel.load(Ordering::Relaxed) {
shared.state.store(STATE_READY, Ordering::Release);
}
}
RecvAct::Batch(cs) => pending.extend(cs),
}
if !pending.is_empty() {
apply_changes(&shared, std::mem::take(&mut pending));
}
}
if let Some(watcher) = watcher {
detach_watcher(watcher);
}
}
fn detach_watcher(watcher: notify::RecommendedWatcher) {
log_unwatch_spawn(
std::thread::Builder::new()
.name("whycodes-index-unwatch".into())
.spawn(move || drop(watcher)),
);
}
pub(crate) fn log_unwatch_spawn(result: std::io::Result<std::thread::JoinHandle<()>>) {
if let Err(e) = result {
tracing::debug!(error = %e, "index unwatch thread spawn failed");
}
}
fn full_scan(shared: &Arc<Shared>) {
shared.scanned.store(0, Ordering::Relaxed);
shared.truncated.store(false, Ordering::Relaxed);
shared.state.store(STATE_SCANNING, Ordering::Release);
for (i, state) in shared.states.iter().enumerate() {
if shared.cancel.load(Ordering::Relaxed) {
return;
}
write(&state.store).clear();
write(&state.content).clear();
let injector = {
let mut fz = lock(&state.fuzzy);
fz.restart();
fz.injector()
};
let remaining = shared
.max_entries
.saturating_sub(shared.scanned.load(Ordering::Relaxed));
if remaining == 0 {
shared.truncated.store(true, Ordering::Relaxed);
break;
}
let queue = Mutex::new(Vec::new());
let stats = walk::walk_root(
&shared.roots[i],
shared.threads,
remaining,
&shared.scanned,
&shared.cancel,
&|e: WalkEntry| {
FuzzyEngine::push(&injector, &e.rel, e.is_dir);
lock(&queue).push(e);
},
);
fill_root(shared, i, std::mem::take(&mut *lock(&queue)));
if stats.truncated {
shared.truncated.store(true, Ordering::Relaxed);
break;
}
}
}
fn fill_root(shared: &Shared, i: usize, entries: Vec<WalkEntry>) {
let state = &shared.states[i];
let mut files = Vec::new();
{
let mut store = write(&state.store);
for e in entries {
if !e.is_dir {
files.push((e.rel.clone(), e.size));
}
store.insert(i as u16, e.rel, e.is_dir, e.size);
}
}
for (rel, size) in files {
if shared.cancel.load(Ordering::Relaxed) {
return;
}
let posting = ContentIndex::load(&shared.roots[i], &rel, size);
write(&state.content).insert(&rel, posting);
}
write(&state.content).mark_complete();
}
pub(crate) fn apply_changes(shared: &Arc<Shared>, changes: Vec<Change>) {
let mut by_root: rustc_hash::FxHashMap<u16, Vec<Change>> = Default::default();
for c in changes {
by_root.entry(c.root).or_default().push(c);
}
for (root_idx, cs) in by_root {
let Some(state) = shared.states.get(root_idx as usize) else {
continue;
};
let root = &shared.roots[root_idx as usize];
let mut saw_removal = false;
let mut upserts: Vec<(Box<str>, bool)> = Vec::new();
{
let mut store = write(&state.store);
let mut content = write(&state.content);
for c in cs {
match c.kind {
ChangeKind::Remove => {
store.remove_tree(root_idx, &c.rel);
content.remove_tree(&c.rel);
saw_removal = true;
}
ChangeKind::Upsert => {
match std::fs::symlink_metadata(root.join(&c.rel)) {
Ok(md) => {
let is_dir = md.is_dir();
let size = if is_dir { 0 } else { md.len() };
upserts.push((c.rel.clone().into_boxed_str(), is_dir));
if is_dir {
content.remove(&c.rel);
} else {
content.upsert(root, &c.rel, size);
}
store.insert(root_idx, c.rel.into_boxed_str(), is_dir, size);
}
Err(_) => {
store.remove_tree(root_idx, &c.rel);
content.remove_tree(&c.rel);
saw_removal = true;
}
}
}
}
}
}
let mut fz = lock(&state.fuzzy);
if saw_removal {
fz.restart();
let inj = fz.injector();
for e in read(&state.store).iter_root(root_idx) {
FuzzyEngine::push(&inj, &e.rel, e.is_dir);
}
} else {
let inj = fz.injector();
for (rel, is_dir) in upserts {
FuzzyEngine::push(&inj, &rel, is_dir);
}
}
}
}
#[cfg(test)]
#[path = "lib_tests.rs"]
mod tests;