fallow_extract/
lib.rs

1//! Parsing and extraction engine for the fallow codebase analyzer.
2//!
3//! This crate handles all file parsing: JS/TS via Oxc, Vue/Svelte SFC extraction,
4//! Astro frontmatter, MDX import/export extraction, CSS Module class name extraction,
5//! HTML asset reference extraction, and incremental caching of parse results.
6
7#![warn(missing_docs)]
8
9pub mod astro;
10pub mod cache;
11pub(crate) mod complexity;
12pub mod css;
13pub mod html;
14pub mod mdx;
15mod parse;
16pub mod sfc;
17mod sfc_template;
18pub mod suppress;
19mod template_usage;
20pub mod visitor;
21
22use std::path::Path;
23
24use rayon::prelude::*;
25
26use cache::CacheStore;
27use fallow_types::discover::{DiscoveredFile, FileId};
28
29// Re-export all extract types from fallow-types
30pub use fallow_types::extract::{
31    DynamicImportInfo, DynamicImportPattern, ExportInfo, ExportName, ImportInfo, ImportedName,
32    MemberAccess, MemberInfo, MemberKind, ModuleInfo, ParseResult, ReExportInfo, RequireCallInfo,
33    compute_line_offsets,
34};
35
36// Re-export extraction functions for internal use and fuzzing
37pub use astro::extract_astro_frontmatter;
38pub use css::extract_css_module_exports;
39pub use mdx::extract_mdx_statements;
40pub use sfc::{extract_sfc_scripts, is_sfc_file};
41
42use parse::parse_source_to_module;
43
44/// Parse all files in parallel, extracting imports and exports.
45/// Uses the cache to skip reparsing files whose content hasn't changed.
46pub fn parse_all_files(files: &[DiscoveredFile], cache: Option<&CacheStore>) -> ParseResult {
47    use std::sync::atomic::{AtomicUsize, Ordering};
48    let cache_hits = AtomicUsize::new(0);
49    let cache_misses = AtomicUsize::new(0);
50
51    let modules: Vec<ModuleInfo> = files
52        .par_iter()
53        .filter_map(|file| parse_single_file_cached(file, cache, &cache_hits, &cache_misses))
54        .collect();
55
56    let hits = cache_hits.load(Ordering::Relaxed);
57    let misses = cache_misses.load(Ordering::Relaxed);
58    if hits > 0 || misses > 0 {
59        tracing::info!(
60            cache_hits = hits,
61            cache_misses = misses,
62            "incremental cache stats"
63        );
64    }
65
66    ParseResult {
67        modules,
68        cache_hits: hits,
69        cache_misses: misses,
70    }
71}
72
73/// Extract mtime (seconds since epoch) from file metadata.
74/// Returns 0 if mtime cannot be determined (pre-epoch, unsupported OS, etc.).
75fn mtime_secs(metadata: &std::fs::Metadata) -> u64 {
76    metadata
77        .modified()
78        .ok()
79        .and_then(|t| t.duration_since(std::time::SystemTime::UNIX_EPOCH).ok())
80        .map_or(0, |d| d.as_secs())
81}
82
83/// Parse a single file, consulting the cache first.
84///
85/// Cache validation strategy (fast path -> slow path):
86/// 1. `stat()` the file to get mtime + size (single syscall, no file read)
87/// 2. If mtime+size match the cached entry -> cache hit, return immediately
88/// 3. If mtime+size differ -> read file, compute content hash
89/// 4. If content hash matches cached entry -> cache hit (file was `touch`ed but unchanged)
90/// 5. Otherwise -> cache miss, full parse
91fn parse_single_file_cached(
92    file: &DiscoveredFile,
93    cache: Option<&CacheStore>,
94    cache_hits: &std::sync::atomic::AtomicUsize,
95    cache_misses: &std::sync::atomic::AtomicUsize,
96) -> Option<ModuleInfo> {
97    use std::sync::atomic::Ordering;
98
99    // Fast path: check mtime+size before reading file content.
100    // A single stat() syscall is ~100x cheaper than read()+hash().
101    if let Some(store) = cache
102        && let Ok(metadata) = std::fs::metadata(&file.path)
103    {
104        let mt = mtime_secs(&metadata);
105        let sz = metadata.len();
106        if let Some(cached) = store.get_by_metadata(&file.path, mt, sz) {
107            cache_hits.fetch_add(1, Ordering::Relaxed);
108            return Some(cache::cached_to_module(cached, file.id));
109        }
110    }
111
112    // Slow path: read file content and compute content hash.
113    let source = std::fs::read_to_string(&file.path).ok()?;
114    let content_hash = xxhash_rust::xxh3::xxh3_64(source.as_bytes());
115
116    // Check cache by content hash (handles touch/save-without-change)
117    if let Some(store) = cache
118        && let Some(cached) = store.get(&file.path, content_hash)
119    {
120        cache_hits.fetch_add(1, Ordering::Relaxed);
121        return Some(cache::cached_to_module(cached, file.id));
122    }
123    cache_misses.fetch_add(1, Ordering::Relaxed);
124
125    // Cache miss — do a full parse
126    Some(parse_source_to_module(
127        file.id,
128        &file.path,
129        &source,
130        content_hash,
131    ))
132}
133
134/// Parse a single file and extract module information.
135#[must_use]
136pub fn parse_single_file(file: &DiscoveredFile) -> Option<ModuleInfo> {
137    let source = std::fs::read_to_string(&file.path).ok()?;
138    let content_hash = xxhash_rust::xxh3::xxh3_64(source.as_bytes());
139    Some(parse_source_to_module(
140        file.id,
141        &file.path,
142        &source,
143        content_hash,
144    ))
145}
146
147/// Parse from in-memory content (for LSP).
148#[must_use]
149pub fn parse_from_content(file_id: FileId, path: &Path, content: &str) -> ModuleInfo {
150    let content_hash = xxhash_rust::xxh3::xxh3_64(content.as_bytes());
151    parse_source_to_module(file_id, path, content, content_hash)
152}
153
154// Parser integration tests invoke Oxc under Miri which is ~1000x slower.
155// Unit tests in individual modules (visitor, suppress, sfc, css, etc.) still run.
156#[cfg(all(test, not(miri)))]
157mod tests;
fallow_extract/lib.rs

fallow_extract/
lib.rs
