fallow_extract/
lib.rs

1//! Parsing and extraction engine for fallow codebase intelligence.
2//!
3//! This crate handles all file parsing: JS/TS via Oxc, Vue/Svelte SFC extraction,
4//! Astro frontmatter, MDX import/export extraction, CSS Module class name extraction,
5//! HTML asset reference extraction, and incremental caching of parse results.
6
7#![warn(missing_docs)]
8#![cfg_attr(not(test), deny(clippy::disallowed_methods))]
9#![cfg_attr(
10    test,
11    allow(
12        clippy::unwrap_used,
13        clippy::expect_used,
14        reason = "tests use unwrap and expect to keep fixture setup concise"
15    )
16)]
17
18mod asset_url;
19pub mod astro;
20pub mod cache;
21pub(crate) mod complexity;
22pub mod css;
23pub mod css_classes;
24pub mod css_in_js;
25pub mod css_metrics;
26pub mod flags;
27pub mod glimmer;
28pub(crate) mod graphql;
29pub(crate) mod html;
30pub(crate) mod iconify;
31pub mod inventory;
32pub mod mdx;
33mod module_info;
34mod parse;
35pub mod sfc;
36pub mod sfc_css;
37mod sfc_props;
38mod sfc_template;
39pub mod similar_code;
40mod source_map;
41pub mod suppress;
42/// Tailwind CSS arbitrary-value detection.
43pub mod tailwind;
44pub(crate) mod template_complexity;
45mod template_expression_scan;
46mod template_usage;
47/// Visitor utilities for AST extraction.
48pub mod visitor;
49
50use std::path::Path;
51use std::sync::atomic::{AtomicBool, Ordering};
52
53use rayon::prelude::*;
54
55use cache::CacheStore;
56use fallow_types::discover::{DiscoveredFile, FileId};
57
58pub use fallow_types::extract::{
59    AngularComponentFieldArrayTypeFact, AngularTemplateMemberAccessFact, AngularThisSpreadFact,
60    ClassHeritageInfo, ClassThisMemberAccessFact, ClassThisWholeObjectUseFact,
61    ComputedEnumKeyUseFact, DefaultImportWholeObjectUseFact, DynamicCustomElementRenderFact,
62    DynamicImportInfo, DynamicImportPattern, ExportInfo, ExportName,
63    ExportedObjectInstancePropertyFact, FactoryCallMemberAccessFact, FactoryFnMemberAccessFact,
64    FactoryFnWholeObjectFact, FactoryReturnExport, FactoryReturnObjectPropertyAccessFact,
65    FactoryReturnObjectShapeExport, FluentChainMemberAccessFact, FluentChainNewMemberAccessFact,
66    ImportInfo, ImportedName, InstanceExportBindingFact, LocalTypeDeclaration, MemberAccess,
67    MemberInfo, MemberKind, ModuleInfo, ModuleLoadMechanism, ParseResult,
68    PlaywrightFixtureAliasFact, PlaywrightFixtureDefinitionFact, PlaywrightFixtureTypeFact,
69    PlaywrightFixtureUseFact, PublicSignatureTypeReference, QualifiedClassMemberAccessFact,
70    ReExportInfo, RequireCallInfo, RequiredTypeMemberFact, SemanticFact, SourceReadFailure,
71    StringEnumMemberValueFact, TypeAliasSurfaceTargetFact, TypeMemberTypeEntry,
72    TypedPropertyMemberAccessFact, VisibilityTag, VitestModuleMockAction,
73    VitestModuleMockOperationFact, compute_line_offsets,
74};
75
76pub use astro::{
77    extract_astro_frontmatter, extract_astro_style_regions, extract_astro_template_regions,
78};
79pub use css::{
80    ThemeScan, ThemeTokenDef, extract_apply_tokens, extract_apply_tokens_located,
81    extract_css_module_exports, extract_css_var_reads_located, scan_theme_blocks,
82};
83pub use css_classes::{
84    MarkupClassScan, MarkupClassToken, is_edit_distance_one, is_typo_edit, scan_markup_class_tokens,
85};
86pub use css_in_js::{
87    ConsumerQuery, CssInJsObjectSheets, CssInJsToken, CssInJsTokenDef, CssInJsTokenOrigin,
88    TokenConsumerHit, css_in_js_consumer_scan, css_in_js_object_sheets, css_in_js_theme_consumers,
89    css_in_js_theme_token_defs, css_in_js_token_consumers, css_in_js_token_defs,
90    css_in_js_virtual_stylesheet, panda_style_value_consumers, panda_token_call_consumers,
91};
92pub use css_metrics::{compute_css_analytics, parse_css_color_rgb};
93pub use glimmer::{is_glimmer_file, strip_glimmer_templates};
94pub use mdx::{extract_mdx_statements, extract_mdx_statements_mapped};
95pub use sfc::{
96    SourceRegion, extract_sfc_scripts, extract_sfc_styles, extract_sfc_template_regions,
97    is_sfc_file,
98};
99pub use sfc_css::{
100    scoped_unused_classes, sfc_preprocessor_virtual_stylesheet, sfc_virtual_stylesheet,
101};
102pub use similar_code::extract_similar_code_functions;
103pub use source_map::ExtractionResult;
104pub use tailwind::{TailwindArbitraryUse, scan_tailwind_arbitrary_values};
105
106#[expect(
107    clippy::expect_used,
108    reason = "static regex patterns are hard-coded analyzer invariants covered by extraction tests"
109)]
110fn static_regex(pattern: &str) -> regex::Regex {
111    regex::Regex::new(pattern).expect("static regex pattern should compile")
112}
113
114pub use parse::parse_source_to_module;
115
116/// Leading UTF-8 byte order mark codepoint.
117///
118/// Windows editors (Notepad, older VS settings, some IDE plugins) emit a UTF-8
119/// BOM at the start of source files. fallow's contract is "UTF-8 with or
120/// without BOM; line offsets are computed against the post-BOM view; the BOM,
121/// if present on input, is preserved on output by `fallow fix`."
122const BOM_CHAR: char = '\u{FEFF}';
123// Small, cache-hot inputs are faster on one thread than through Rayon setup.
124// Larger file sets still use parallel parsing where parse work dominates.
125const PARALLEL_PARSE_FILE_THRESHOLD: usize = 32;
126
127/// Strip the leading UTF-8 BOM if present.
128///
129/// Called at every file-read entry point in this crate so the rest of the
130/// pipeline (content hash, `compute_line_offsets`, oxc parser, downstream
131/// analyses) sees a consistent post-BOM view. Mirrors the
132/// `fallow_config` layer (`config_writer.rs::BOM`) so config-shaped sources
133/// and source-code-shaped sources are processed symmetrically. See issue #475.
134#[must_use]
135fn strip_bom(source: &str) -> &str {
136    source.strip_prefix(BOM_CHAR).unwrap_or(source)
137}
138
139/// Parse all files, extracting imports and exports.
140///
141/// Small file sets use a sequential fast path to avoid parallel scheduling
142/// overhead; larger file sets use parallel extraction.
143/// Uses the cache to skip reparsing files whose content hasn't changed.
144///
145/// When `need_complexity` is true, per-function cyclomatic/cognitive complexity
146/// metrics are computed during parsing (needed by the `health` command).
147/// Pass `false` for dead-code analysis where complexity data is unused.
148pub fn parse_all_files(
149    files: &[DiscoveredFile],
150    cache: Option<&CacheStore>,
151    need_complexity: bool,
152) -> ParseResult {
153    parse_all_files_cancellable(files, cache, need_complexity, None)
154}
155
156/// Parse all files, abandoning the remaining ones once `cancellation` is set.
157///
158/// Rayon's `map`/`collect` cannot short-circuit, so cancellation makes the
159/// per-file body a no-op instead of stopping the iteration: the scheduled
160/// items still drain, but at one atomic load each. The returned
161/// [`ParseResult`] is therefore truncated whenever the token flipped, and
162/// callers must treat a set token as a failed run rather than as a project
163/// with fewer modules.
164pub fn parse_all_files_cancellable(
165    files: &[DiscoveredFile],
166    cache: Option<&CacheStore>,
167    need_complexity: bool,
168    cancellation: Option<&AtomicBool>,
169) -> ParseResult {
170    let parse_one = |file: &DiscoveredFile| {
171        if cancellation.is_some_and(|cancelled| cancelled.load(Ordering::SeqCst)) {
172            return ParseFileResult::default();
173        }
174        parse_single_file_cached(file, cache, need_complexity)
175    };
176    let results: Vec<ParseFileResult> = if files.len() <= PARALLEL_PARSE_FILE_THRESHOLD {
177        files.iter().map(parse_one).collect()
178    } else {
179        files.par_iter().map(parse_one).collect()
180    };
181
182    let mut modules = Vec::with_capacity(results.len());
183    let mut read_failures = Vec::new();
184    let mut hits = 0usize;
185    let mut misses = 0usize;
186    let mut parse_cpu_nanos = 0u64;
187
188    for result in results {
189        hits += result.cache_hits;
190        misses += result.cache_misses;
191        parse_cpu_nanos = parse_cpu_nanos.saturating_add(result.parse_cpu_nanos);
192        if let Some(module) = result.module {
193            modules.push(module);
194        }
195        if let Some(failure) = result.read_failure {
196            read_failures.push(failure);
197        }
198    }
199
200    if hits > 0 || misses > 0 {
201        tracing::info!(
202            cache_hits = hits,
203            cache_misses = misses,
204            "incremental cache stats"
205        );
206    }
207
208    ParseResult {
209        modules,
210        read_failures,
211        cache_hits: hits,
212        cache_misses: misses,
213        parse_cpu_ms: parse_cpu_nanos as f64 / 1_000_000.0,
214    }
215}
216
217#[derive(Default)]
218struct ParseFileResult {
219    module: Option<ModuleInfo>,
220    read_failure: Option<SourceReadFailure>,
221    cache_hits: usize,
222    cache_misses: usize,
223    parse_cpu_nanos: u64,
224}
225
226impl ParseFileResult {
227    fn cache_hit(module: ModuleInfo) -> Self {
228        Self {
229            module: Some(module),
230            read_failure: None,
231            cache_hits: 1,
232            cache_misses: 0,
233            parse_cpu_nanos: 0,
234        }
235    }
236
237    fn cache_miss(module: ModuleInfo, parse_cpu_nanos: u64) -> Self {
238        Self {
239            module: Some(module),
240            read_failure: None,
241            cache_hits: 0,
242            cache_misses: 1,
243            parse_cpu_nanos,
244        }
245    }
246
247    fn read_failure(file: &DiscoveredFile, error: &std::io::Error) -> Self {
248        Self {
249            module: None,
250            read_failure: Some(SourceReadFailure {
251                file_id: file.id,
252                path: file.path.clone(),
253                error: error.to_string(),
254            }),
255            cache_hits: 0,
256            cache_misses: 0,
257            parse_cpu_nanos: 0,
258        }
259    }
260}
261
262/// Parse a single file, consulting the cache first.
263///
264/// Cache validation strategy (fast path -> slow path):
265/// 1. Open the file so unreadable sources cannot use stale cached analysis
266/// 2. Read mtime + size from the open handle
267/// 3. If mtime+size match the cached entry -> cache hit, return immediately
268/// 4. If mtime+size differ -> read file, compute content hash
269/// 5. If content hash matches cached entry -> cache hit (file was `touch`ed but unchanged)
270/// 6. Otherwise -> cache miss, full parse
271fn parse_single_file_cached(
272    file: &DiscoveredFile,
273    cache: Option<&CacheStore>,
274    need_complexity: bool,
275) -> ParseFileResult {
276    let cached_by_path = cache.and_then(|store| store.get_by_path_only(&file.path));
277
278    if let Some(cached) = cached_by_path
279        && cached.file_size == file.size_bytes
280    {
281        let source_file = match std::fs::File::open(&file.path) {
282            Ok(source_file) => source_file,
283            Err(error) => return ParseFileResult::read_failure(file, &error),
284        };
285        if let Ok(metadata) = source_file.metadata()
286            && metadata.len() == cached.file_size
287        {
288            let fingerprint =
289                fallow_types::source_fingerprint::SourceFingerprint::from_metadata(&metadata);
290            if cached.source_fingerprint() == fingerprint
291                && fingerprint.has_known_mtime()
292                && (!need_complexity || !cached.complexity.is_empty())
293            {
294                return ParseFileResult::cache_hit(cache::cached_to_module_opts(
295                    cached,
296                    file.id,
297                    need_complexity,
298                ));
299            }
300        }
301    }
302
303    let raw = match std::fs::read_to_string(&file.path) {
304        Ok(raw) => raw,
305        Err(error) => return ParseFileResult::read_failure(file, &error),
306    };
307    let source = strip_bom(&raw);
308    let content_hash = xxhash_rust::xxh3::xxh3_64(source.as_bytes());
309
310    if let Some(cached) = cached_by_path
311        && cached.content_hash == content_hash
312        && (!need_complexity || !cached.complexity.is_empty())
313    {
314        return ParseFileResult::cache_hit(cache::cached_to_module_opts(
315            cached,
316            file.id,
317            need_complexity,
318        ));
319    }
320
321    let parse_start = std::time::Instant::now();
322    let module = parse_source_to_module(file.id, &file.path, source, content_hash, need_complexity);
323    let parse_cpu_nanos = u64::try_from(parse_start.elapsed().as_nanos()).unwrap_or(u64::MAX);
324    ParseFileResult::cache_miss(module, parse_cpu_nanos)
325}
326
327/// Parse a single file and extract module information (without complexity).
328#[must_use]
329pub fn parse_single_file(file: &DiscoveredFile) -> Option<ModuleInfo> {
330    let raw = std::fs::read_to_string(&file.path).ok()?;
331    let source = strip_bom(&raw);
332    let content_hash = xxhash_rust::xxh3::xxh3_64(source.as_bytes());
333    Some(parse_source_to_module(
334        file.id,
335        &file.path,
336        source,
337        content_hash,
338        false,
339    ))
340}
341
342/// Parse from in-memory content (for LSP, includes complexity).
343#[must_use]
344pub fn parse_from_content(file_id: FileId, path: &Path, content: &str) -> ModuleInfo {
345    let content = strip_bom(content);
346    let content_hash = xxhash_rust::xxh3::xxh3_64(content.as_bytes());
347    parse_source_to_module(file_id, path, content, content_hash, true)
348}
349
350#[cfg(all(test, not(miri)))]
351mod tests;
fallow_extract/lib.rs

fallow_extract/
lib.rs
