1mod content;
23mod fuzzy;
24pub mod policy;
25mod store;
26pub mod walk;
27mod watch;
28
29use std::path::{Path, PathBuf};
30use std::sync::atomic::{AtomicBool, AtomicU8, AtomicUsize, Ordering};
31use std::sync::mpsc::{self, Receiver, Sender};
32use std::sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard};
33use std::thread::JoinHandle;
34use std::time::{Duration, Instant};
35
36pub use content::{
37 ContentIndex, MAX_INDEX_BYTES, TRIGRAM_LEN, extract_trigrams, pack_trigram, required_trigrams,
38};
39pub use fuzzy::FileMatch;
40use fuzzy::FuzzyEngine;
41pub use store::{Entry, IndexStore};
42pub use walk::{WalkEntry, WalkStats, walk_root, walk_root_limited};
43use watch::{Change, ChangeKind, Command};
44
45fn project_meta_dir(working_dir: &Path) -> PathBuf {
49 working_dir.join(".whycodes")
50}
51
52pub const DEFAULT_MAX_ENTRIES: usize = 200_000;
54
55const WATCH_DEBOUNCE: Duration = Duration::from_millis(250);
57
58const STATE_SCANNING: u8 = 0;
59const STATE_READY: u8 = 1;
60
61#[derive(Debug, Clone, Copy, PartialEq, Eq)]
63pub enum ScanStatus {
64 Scanning {
66 scanned: usize,
68 },
69 Ready {
71 total: usize,
73 truncated: bool,
75 },
76}
77
78#[derive(Debug, Clone)]
80pub struct IndexOptions {
81 pub watch: bool,
83 pub max_entries: usize,
85 pub threads: usize,
87}
88
89impl Default for IndexOptions {
90 fn default() -> Self {
91 Self {
92 watch: true,
93 max_entries: DEFAULT_MAX_ENTRIES,
94 threads: 0,
95 }
96 }
97}
98
99struct RootState {
101 store: RwLock<IndexStore>,
102 fuzzy: Mutex<FuzzyEngine>,
103 content: RwLock<ContentIndex>,
104}
105
106struct Shared {
108 roots: Vec<PathBuf>,
109 states: Vec<RootState>,
110 scanned: AtomicUsize,
111 state: AtomicU8,
112 truncated: AtomicBool,
113 cancel: AtomicBool,
114 cmd_tx: Sender<Command>,
115 results_dirty: Arc<AtomicBool>,
118 threads: usize,
119 max_entries: usize,
120 watch: bool,
121}
122
123fn lock<T>(m: &Mutex<T>) -> MutexGuard<'_, T> {
124 m.lock().unwrap_or_else(|e| e.into_inner())
125}
126fn read<T>(l: &RwLock<T>) -> RwLockReadGuard<'_, T> {
127 l.read().unwrap_or_else(|e| e.into_inner())
128}
129fn write<T>(l: &RwLock<T>) -> RwLockWriteGuard<'_, T> {
130 l.write().unwrap_or_else(|e| e.into_inner())
131}
132
133pub struct WorkspaceIndex {
139 shared: Arc<Shared>,
140 scanner: Option<JoinHandle<()>>,
141}
142
143impl WorkspaceIndex {
144 pub fn start(roots: Vec<PathBuf>) -> Arc<Self> {
146 Self::start_with(roots, IndexOptions::default())
147 }
148
149 pub fn start_with(roots: Vec<PathBuf>, opts: IndexOptions) -> Arc<Self> {
154 let roots = sanitize_roots(roots);
155 let results_dirty = Arc::new(AtomicBool::new(false));
156 let states = (0..roots.len())
157 .map(|_| RootState {
158 store: RwLock::new(IndexStore::new()),
159 fuzzy: Mutex::new(FuzzyEngine::new(results_dirty.clone())),
160 content: RwLock::new(ContentIndex::new()),
161 })
162 .collect();
163 let (cmd_tx, cmd_rx) = mpsc::channel();
164 let threads = if opts.threads == 0 {
165 std::thread::available_parallelism()
166 .map(|n| n.get().min(8))
167 .unwrap_or(1)
168 } else {
169 opts.threads
170 };
171 let shared = Arc::new(Shared {
172 roots,
173 states,
174 scanned: AtomicUsize::new(0),
175 state: AtomicU8::new(STATE_SCANNING),
176 truncated: AtomicBool::new(false),
177 cancel: AtomicBool::new(false),
178 cmd_tx,
179 results_dirty,
180 threads,
181 max_entries: opts.max_entries,
182 watch: opts.watch,
183 });
184 let thread_shared = shared.clone();
185 let scanner = std::thread::Builder::new()
186 .name("whycodes-index".into())
187 .spawn(move || scanner_main(thread_shared, cmd_rx))
188 .ok();
189 Arc::new(Self { shared, scanner })
190 }
191
192 pub fn project_roots(working_dir: &Path) -> Vec<PathBuf> {
195 let mut roots = vec![working_dir.to_path_buf()];
196 let allow = project_meta_dir(working_dir).join("external_dirs_allowed");
197 if let Ok(content) = std::fs::read_to_string(&allow) {
198 for line in content.lines() {
199 let line = line.trim();
200 if line.is_empty() || line.starts_with('#') {
201 continue;
202 }
203 roots.push(PathBuf::from(line));
204 }
205 }
206 roots
207 }
208
209 pub fn roots(&self) -> &[PathBuf] {
211 &self.shared.roots
212 }
213
214 pub fn primary_root(&self) -> Option<&Path> {
216 self.shared.roots.first().map(PathBuf::as_path)
217 }
218
219 pub fn status(&self) -> ScanStatus {
221 if self.shared.state.load(Ordering::Acquire) == STATE_READY {
222 ScanStatus::Ready {
223 total: self.len(),
224 truncated: self.shared.truncated.load(Ordering::Relaxed),
225 }
226 } else {
227 ScanStatus::Scanning {
228 scanned: self.shared.scanned.load(Ordering::Relaxed),
229 }
230 }
231 }
232
233 pub fn is_ready(&self) -> bool {
235 self.shared.state.load(Ordering::Acquire) == STATE_READY
236 }
237
238 pub fn len(&self) -> usize {
240 self.shared
241 .states
242 .iter()
243 .map(|s| read(&s.store).len())
244 .sum()
245 }
246
247 pub fn is_empty(&self) -> bool {
249 self.len() == 0
250 }
251
252 pub fn wait_ready(&self, timeout: Duration) -> bool {
255 let deadline = Instant::now() + timeout;
256 while !self.is_ready() && Instant::now() < deadline {
257 std::thread::sleep(Duration::from_millis(2));
258 }
259 self.is_ready()
260 }
261
262 pub fn query_now(&self, pattern: &str, limit: usize) -> Vec<FileMatch> {
271 let pattern = pattern.trim();
272 if pattern.is_empty() {
273 return self.browse(0, "");
274 }
275 if let Some(dir) = pattern.strip_suffix('/') {
276 return self.browse(0, dir);
277 }
278 for state in &self.shared.states {
279 lock(&state.fuzzy).set_query(pattern);
280 }
281 self.read_fuzzy(limit).0
282 }
283
284 pub fn read_matches(&self, limit: usize) -> Vec<FileMatch> {
287 self.read_fuzzy(limit).0
288 }
289
290 pub fn take_results_dirty(&self) -> bool {
292 self.shared.results_dirty.swap(false, Ordering::Acquire)
293 }
294
295 pub fn rearm_fuzzy(&self) {
300 for state in &self.shared.states {
301 lock(&state.fuzzy).rearm();
302 }
303 }
304
305 pub fn matching(&self) -> bool {
308 self.shared.states.iter().any(|s| lock(&s.fuzzy).nudge())
309 }
310
311 fn read_fuzzy(&self, limit: usize) -> (Vec<FileMatch>, bool) {
313 let mut all = Vec::new();
314 let mut running = false;
315 for (i, state) in self.shared.states.iter().enumerate() {
316 let (mut hits, r) = lock(&state.fuzzy).read(limit);
317 running |= r;
318 for h in &mut hits {
319 h.root = i as u16;
320 }
321 all.extend(hits);
322 }
323 all.sort_by(|a, b| {
324 b.score
325 .cmp(&a.score)
326 .then_with(|| a.rel.len().cmp(&b.rel.len()))
327 });
328 all.truncate(limit);
329 (all, running)
330 }
331
332 pub fn query(&self, pattern: &str, limit: usize) -> Vec<FileMatch> {
335 let pattern = pattern.trim();
336 if pattern.is_empty() {
337 return self.browse(0, "");
338 }
339 if let Some(dir) = pattern.strip_suffix('/') {
340 return self.browse(0, dir);
341 }
342 let mut all = Vec::new();
343 for (i, state) in self.shared.states.iter().enumerate() {
344 let mut hits = lock(&state.fuzzy).query_blocking(pattern, limit);
345 for h in &mut hits {
346 h.root = i as u16;
347 }
348 all.extend(hits);
349 }
350 all.sort_by(|a, b| {
351 b.score
352 .cmp(&a.score)
353 .then_with(|| a.rel.len().cmp(&b.rel.len()))
354 });
355 all.truncate(limit);
356 all
357 }
358
359 pub fn browse(&self, root: u16, rel_dir: &str) -> Vec<FileMatch> {
362 let Some(state) = self.shared.states.get(root as usize) else {
363 return Vec::new();
364 };
365 read(&state.store)
366 .browse(root, rel_dir)
367 .into_iter()
368 .map(|e| FileMatch {
369 rel: e.rel.to_string(),
370 is_dir: e.is_dir,
371 score: 0,
372 indices: Vec::new(),
373 root,
374 })
375 .collect()
376 }
377
378 pub fn visit(&self, f: &mut dyn FnMut(&Entry) -> bool) {
381 if let Some(state) = self.shared.states.first() {
382 for e in read(&state.store).entries() {
383 if !f(e) {
384 return;
385 }
386 }
387 }
388 }
389
390 pub fn visit_content_candidates(
398 &self,
399 search_root: &Path,
400 pattern: &str,
401 case_insensitive: bool,
402 f: &mut dyn FnMut(&Path, &str, u64) -> bool,
403 ) -> Option<()> {
404 let required = required_trigrams(pattern, case_insensitive)?;
405 let state = self.shared.states.first()?;
406 let store = read(&state.store);
408 let content = read(&state.content);
409 if !content.is_complete() {
410 return None;
411 }
412 let root = self.shared.roots.first()?;
413 let prefix = content_search_prefix(root, search_root)?;
414 for e in store.entries() {
415 if e.is_dir || !content_entry_in_scope(&prefix, &e.rel) {
416 continue;
417 }
418 if !content.may_match(&e.rel, &required) {
419 continue;
420 }
421 let rel = content_scoped_rel(&prefix, &e.rel);
422 if !f(&root.join(&*e.rel), &rel, e.size) {
423 break;
424 }
425 }
426 Some(())
427 }
428
429 pub fn entries(&self) -> Vec<Entry> {
431 self.shared
432 .states
433 .first()
434 .map(|s| read(&s.store).entries().to_vec())
435 .unwrap_or_default()
436 }
437
438 pub fn resolve(&self, m: &FileMatch) -> PathBuf {
440 self.shared.roots[m.root as usize].join(&m.rel)
441 }
442
443 pub fn rescan(&self) {
445 let _ = self.shared.cmd_tx.send(Command::Rescan);
446 }
447}
448
449impl Drop for WorkspaceIndex {
450 fn drop(&mut self) {
451 self.shared.cancel.store(true, Ordering::Relaxed);
452 let _ = self.shared.cmd_tx.send(Command::Shutdown);
453 if let Some(h) = self.scanner.take() {
454 let _ = h.join();
457 }
458 }
459}
460
461impl std::fmt::Debug for WorkspaceIndex {
462 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
463 f.debug_struct("WorkspaceIndex")
464 .field("roots", &self.shared.roots)
465 .field("status", &self.status())
466 .finish()
467 }
468}
469
470fn content_search_prefix(primary: &Path, search_root: &Path) -> Option<String> {
471 let search = std::fs::canonicalize(search_root).unwrap_or_else(|_| search_root.to_path_buf());
472 let rel = search.strip_prefix(primary).ok()?;
473 let prefix = rel.to_string_lossy().replace('\\', "/");
474 Some(prefix.trim_matches('/').to_string())
475}
476
477fn content_entry_in_scope(prefix: &str, rel: &str) -> bool {
478 prefix.is_empty()
479 || (rel.len() > prefix.len()
480 && rel.starts_with(prefix)
481 && rel.as_bytes().get(prefix.len()) == Some(&b'/'))
482}
483
484fn content_scoped_rel(prefix: &str, rel: &str) -> String {
485 if prefix.is_empty() {
486 rel.to_string()
487 } else {
488 rel[prefix.len() + 1..].to_string()
489 }
490}
491
492fn sanitize_roots(roots: Vec<PathBuf>) -> Vec<PathBuf> {
494 let mut out: Vec<PathBuf> = Vec::new();
495 for root in roots {
496 let Ok(canon) = std::fs::canonicalize(&root) else {
497 continue;
498 };
499 if !canon.is_dir() {
500 continue;
501 }
502 if out.iter().any(|r| canon.starts_with(r)) {
504 continue;
505 }
506 out.push(canon);
507 }
508 out
509}
510
511#[cfg(test)]
512pub(crate) fn recv_should_stop(e: mpsc::RecvTimeoutError) -> bool {
513 matches!(e, mpsc::RecvTimeoutError::Disconnected)
514}
515
516#[derive(Debug)]
517pub(crate) enum RecvAct {
518 Stop,
519 Idle,
520 Rescan,
521 Batch(Vec<Change>),
522}
523
524pub(crate) fn classify_recv(r: Result<Command, mpsc::RecvTimeoutError>) -> RecvAct {
525 match r {
526 Ok(Command::Shutdown) => RecvAct::Stop,
527 Ok(Command::Rescan) => RecvAct::Rescan,
528 Ok(Command::Batch(cs)) => RecvAct::Batch(cs),
529 Err(mpsc::RecvTimeoutError::Disconnected) => RecvAct::Stop,
530 Err(mpsc::RecvTimeoutError::Timeout) => RecvAct::Idle,
531 }
532}
533
534fn scanner_main(shared: Arc<Shared>, cmd_rx: Receiver<Command>) {
536 full_scan(&shared);
537 if shared.cancel.load(Ordering::Relaxed) {
538 return;
539 }
540
541 let watcher = if shared.watch {
545 watch::spawn(&shared.roots, shared.cmd_tx.clone())
546 } else {
547 None
548 };
549 shared.state.store(STATE_READY, Ordering::Release);
550
551 let mut pending: Vec<Change> = Vec::new();
552 loop {
553 if shared.cancel.load(Ordering::Relaxed) {
554 break;
555 }
556 match classify_recv(cmd_rx.recv_timeout(WATCH_DEBOUNCE)) {
557 RecvAct::Stop => break,
558 RecvAct::Idle => {}
559 RecvAct::Rescan => {
560 pending.clear();
561 full_scan(&shared);
562 if !shared.cancel.load(Ordering::Relaxed) {
563 shared.state.store(STATE_READY, Ordering::Release);
564 }
565 }
566 RecvAct::Batch(cs) => pending.extend(cs),
567 }
568 if !pending.is_empty() {
569 apply_changes(&shared, std::mem::take(&mut pending));
570 }
571 }
572 if let Some(watcher) = watcher {
575 detach_watcher(watcher);
576 }
577}
578
579fn detach_watcher(watcher: notify::RecommendedWatcher) {
582 log_unwatch_spawn(
583 std::thread::Builder::new()
584 .name("whycodes-index-unwatch".into())
585 .spawn(move || drop(watcher)),
586 );
587}
588
589pub(crate) fn log_unwatch_spawn(result: std::io::Result<std::thread::JoinHandle<()>>) {
590 if let Err(e) = result {
591 tracing::debug!(error = %e, "index unwatch thread spawn failed");
592 }
593}
594
595fn full_scan(shared: &Arc<Shared>) {
598 shared.scanned.store(0, Ordering::Relaxed);
599 shared.truncated.store(false, Ordering::Relaxed);
600 shared.state.store(STATE_SCANNING, Ordering::Release);
601
602 for (i, state) in shared.states.iter().enumerate() {
603 if shared.cancel.load(Ordering::Relaxed) {
604 return;
605 }
606 write(&state.store).clear();
607 write(&state.content).clear();
608 let injector = {
609 let mut fz = lock(&state.fuzzy);
610 fz.restart();
611 fz.injector()
612 };
613 let remaining = shared
614 .max_entries
615 .saturating_sub(shared.scanned.load(Ordering::Relaxed));
616 if remaining == 0 {
617 shared.truncated.store(true, Ordering::Relaxed);
618 break;
619 }
620 let queue = Mutex::new(Vec::new());
623 let stats = walk::walk_root(
624 &shared.roots[i],
625 shared.threads,
626 remaining,
627 &shared.scanned,
628 &shared.cancel,
629 &|e: WalkEntry| {
630 FuzzyEngine::push(&injector, &e.rel, e.is_dir);
631 lock(&queue).push(e);
632 },
633 );
634 fill_root(shared, i, std::mem::take(&mut *lock(&queue)));
636 if stats.truncated {
637 shared.truncated.store(true, Ordering::Relaxed);
638 break;
639 }
640 }
641}
642
643fn fill_root(shared: &Shared, i: usize, entries: Vec<WalkEntry>) {
651 let state = &shared.states[i];
652 let mut files = Vec::new();
653 {
654 let mut store = write(&state.store);
655 for e in entries {
656 if !e.is_dir {
657 files.push((e.rel.clone(), e.size));
658 }
659 store.insert(i as u16, e.rel, e.is_dir, e.size);
660 }
661 }
662 for (rel, size) in files {
663 if shared.cancel.load(Ordering::Relaxed) {
664 return;
665 }
666 let posting = ContentIndex::load(&shared.roots[i], &rel, size);
667 write(&state.content).insert(&rel, posting);
668 }
669 write(&state.content).mark_complete();
670}
671
672pub(crate) fn apply_changes(shared: &Arc<Shared>, changes: Vec<Change>) {
675 let mut by_root: rustc_hash::FxHashMap<u16, Vec<Change>> = Default::default();
677 for c in changes {
678 by_root.entry(c.root).or_default().push(c);
679 }
680 for (root_idx, cs) in by_root {
681 let Some(state) = shared.states.get(root_idx as usize) else {
682 continue;
683 };
684 let root = &shared.roots[root_idx as usize];
685 let mut saw_removal = false;
686 let mut upserts: Vec<(Box<str>, bool)> = Vec::new();
687 {
688 let mut store = write(&state.store);
689 let mut content = write(&state.content);
690 for c in cs {
691 match c.kind {
692 ChangeKind::Remove => {
693 store.remove_tree(root_idx, &c.rel);
694 content.remove_tree(&c.rel);
695 saw_removal = true;
696 }
697 ChangeKind::Upsert => {
698 match std::fs::symlink_metadata(root.join(&c.rel)) {
701 Ok(md) => {
702 let is_dir = md.is_dir();
703 let size = if is_dir { 0 } else { md.len() };
704 upserts.push((c.rel.clone().into_boxed_str(), is_dir));
705 if is_dir {
706 content.remove(&c.rel);
707 } else {
708 content.upsert(root, &c.rel, size);
709 }
710 store.insert(root_idx, c.rel.into_boxed_str(), is_dir, size);
711 }
712 Err(_) => {
713 store.remove_tree(root_idx, &c.rel);
714 content.remove_tree(&c.rel);
715 saw_removal = true;
716 }
717 }
718 }
719 }
720 }
721 }
722 let mut fz = lock(&state.fuzzy);
725 if saw_removal {
726 fz.restart();
727 let inj = fz.injector();
728 for e in read(&state.store).iter_root(root_idx) {
729 FuzzyEngine::push(&inj, &e.rel, e.is_dir);
730 }
731 } else {
732 let inj = fz.injector();
733 for (rel, is_dir) in upserts {
734 FuzzyEngine::push(&inj, &rel, is_dir);
735 }
736 }
737 }
738}
739
740#[cfg(test)]
741#[path = "lib_tests.rs"]
742mod tests;