fallow_extract/
lib.rs

1//! Parsing and extraction engine for the fallow codebase analyzer.
2//!
3//! This crate handles all file parsing: JS/TS via Oxc, Vue/Svelte SFC extraction,
4//! Astro frontmatter, MDX import/export extraction, CSS Module class name extraction,
5//! and incremental caching of parse results.
6
7#![warn(missing_docs)]
8
9pub mod astro;
10pub mod cache;
11pub mod css;
12pub mod mdx;
13mod parse;
14pub mod sfc;
15pub mod suppress;
16pub mod visitor;
17
18use std::path::Path;
19
20use rayon::prelude::*;
21
22use cache::CacheStore;
23use fallow_types::discover::{DiscoveredFile, FileId};
24
25// Re-export all extract types from fallow-types
26pub use fallow_types::extract::{
27    DynamicImportInfo, DynamicImportPattern, ExportInfo, ExportName, ImportInfo, ImportedName,
28    MemberAccess, MemberInfo, MemberKind, ModuleInfo, ParseResult, ReExportInfo, RequireCallInfo,
29    compute_line_offsets,
30};
31
32// Re-export extraction functions for internal use and fuzzing
33pub use astro::extract_astro_frontmatter;
34pub use css::extract_css_module_exports;
35pub use mdx::extract_mdx_statements;
36pub use sfc::{extract_sfc_scripts, is_sfc_file};
37
38use parse::parse_source_to_module;
39
40/// Parse all files in parallel, extracting imports and exports.
41/// Uses the cache to skip reparsing files whose content hasn't changed.
42pub fn parse_all_files(files: &[DiscoveredFile], cache: Option<&CacheStore>) -> ParseResult {
43    use std::sync::atomic::{AtomicUsize, Ordering};
44    let cache_hits = AtomicUsize::new(0);
45    let cache_misses = AtomicUsize::new(0);
46
47    let modules: Vec<ModuleInfo> = files
48        .par_iter()
49        .filter_map(|file| parse_single_file_cached(file, cache, &cache_hits, &cache_misses))
50        .collect();
51
52    let hits = cache_hits.load(Ordering::Relaxed);
53    let misses = cache_misses.load(Ordering::Relaxed);
54    if hits > 0 || misses > 0 {
55        tracing::info!(
56            cache_hits = hits,
57            cache_misses = misses,
58            "incremental cache stats"
59        );
60    }
61
62    ParseResult {
63        modules,
64        cache_hits: hits,
65        cache_misses: misses,
66    }
67}
68
69/// Extract mtime (seconds since epoch) from file metadata.
70/// Returns 0 if mtime cannot be determined (pre-epoch, unsupported OS, etc.).
71fn mtime_secs(metadata: &std::fs::Metadata) -> u64 {
72    metadata
73        .modified()
74        .ok()
75        .and_then(|t| t.duration_since(std::time::SystemTime::UNIX_EPOCH).ok())
76        .map_or(0, |d| d.as_secs())
77}
78
79/// Parse a single file, consulting the cache first.
80///
81/// Cache validation strategy (fast path -> slow path):
82/// 1. `stat()` the file to get mtime + size (single syscall, no file read)
83/// 2. If mtime+size match the cached entry -> cache hit, return immediately
84/// 3. If mtime+size differ -> read file, compute content hash
85/// 4. If content hash matches cached entry -> cache hit (file was `touch`ed but unchanged)
86/// 5. Otherwise -> cache miss, full parse
87fn parse_single_file_cached(
88    file: &DiscoveredFile,
89    cache: Option<&CacheStore>,
90    cache_hits: &std::sync::atomic::AtomicUsize,
91    cache_misses: &std::sync::atomic::AtomicUsize,
92) -> Option<ModuleInfo> {
93    use std::sync::atomic::Ordering;
94
95    // Fast path: check mtime+size before reading file content.
96    // A single stat() syscall is ~100x cheaper than read()+hash().
97    if let Some(store) = cache
98        && let Ok(metadata) = std::fs::metadata(&file.path)
99    {
100        let mt = mtime_secs(&metadata);
101        let sz = metadata.len();
102        if let Some(cached) = store.get_by_metadata(&file.path, mt, sz) {
103            cache_hits.fetch_add(1, Ordering::Relaxed);
104            return Some(cache::cached_to_module(cached, file.id));
105        }
106    }
107
108    // Slow path: read file content and compute content hash.
109    let source = std::fs::read_to_string(&file.path).ok()?;
110    let content_hash = xxhash_rust::xxh3::xxh3_64(source.as_bytes());
111
112    // Check cache by content hash (handles touch/save-without-change)
113    if let Some(store) = cache
114        && let Some(cached) = store.get(&file.path, content_hash)
115    {
116        cache_hits.fetch_add(1, Ordering::Relaxed);
117        return Some(cache::cached_to_module(cached, file.id));
118    }
119    cache_misses.fetch_add(1, Ordering::Relaxed);
120
121    // Cache miss — do a full parse
122    Some(parse_source_to_module(
123        file.id,
124        &file.path,
125        &source,
126        content_hash,
127    ))
128}
129
130/// Parse a single file and extract module information.
131pub fn parse_single_file(file: &DiscoveredFile) -> Option<ModuleInfo> {
132    let source = std::fs::read_to_string(&file.path).ok()?;
133    let content_hash = xxhash_rust::xxh3::xxh3_64(source.as_bytes());
134    Some(parse_source_to_module(
135        file.id,
136        &file.path,
137        &source,
138        content_hash,
139    ))
140}
141
142/// Parse from in-memory content (for LSP).
143pub fn parse_from_content(file_id: FileId, path: &Path, content: &str) -> ModuleInfo {
144    let content_hash = xxhash_rust::xxh3::xxh3_64(content.as_bytes());
145    parse_source_to_module(file_id, path, content, content_hash)
146}
147
148// Parser integration tests invoke Oxc under Miri which is ~1000x slower.
149// Unit tests in individual modules (visitor, suppress, sfc, css, etc.) still run.
150#[cfg(all(test, not(miri)))]
151mod tests;
fallow_extract/lib.rs

fallow_extract/
lib.rs
