fallow_extract/
lib.rs

1//! Parsing and extraction engine for fallow codebase intelligence.
2//!
3//! This crate handles all file parsing: JS/TS via Oxc, Vue/Svelte SFC extraction,
4//! Astro frontmatter, MDX import/export extraction, CSS Module class name extraction,
5//! HTML asset reference extraction, and incremental caching of parse results.
6
7#![warn(missing_docs)]
8#![cfg_attr(not(test), deny(clippy::disallowed_methods))]
9#![cfg_attr(
10    test,
11    allow(
12        clippy::unwrap_used,
13        clippy::expect_used,
14        reason = "tests use unwrap and expect to keep fixture setup concise"
15    )
16)]
17
18pub mod alchemy;
19mod asset_url;
20pub mod astro;
21mod binding_references;
22pub mod cache;
23pub(crate) mod complexity;
24mod complexity_names;
25mod component_contracts;
26pub mod css;
27pub mod css_classes;
28pub mod css_in_js;
29pub mod css_metrics;
30pub mod effect_facts;
31pub mod federation_runtime;
32pub mod flags;
33mod function_body;
34pub mod glimmer;
35pub(crate) mod graphql;
36pub(crate) mod html;
37pub(crate) mod iconify;
38pub mod inventory;
39mod jsdoc_attach;
40mod jsdoc_deprecated;
41pub mod mdx;
42mod module_info;
43pub mod og_image;
44mod parse;
45pub mod sfc;
46pub mod sfc_css;
47mod sfc_props;
48mod sfc_template;
49pub mod similar_code;
50mod source_map;
51pub mod suppress;
52/// Tailwind CSS arbitrary-value detection.
53pub mod tailwind;
54pub(crate) mod template_complexity;
55mod template_expression_scan;
56mod template_usage;
57/// Visitor utilities for AST extraction.
58pub mod visitor;
59
60use std::path::Path;
61use std::sync::atomic::{AtomicBool, Ordering};
62
63use rayon::prelude::*;
64
65use cache::CacheStore;
66use fallow_types::discover::{DiscoveredFile, FileId};
67
68pub use fallow_types::extract::{
69    AngularComponentFieldArrayTypeFact, AngularTemplateMemberAccessFact, AngularThisSpreadFact,
70    ClassHeritageInfo, ClassThisMemberAccessFact, ClassThisWholeObjectUseFact,
71    ComputedEnumKeyUseFact, DefaultImportWholeObjectUseFact, DynamicCustomElementRenderFact,
72    DynamicImportInfo, DynamicImportPattern, ExportInfo, ExportName,
73    ExportedObjectInstancePropertyFact, FactoryCallMemberAccessFact, FactoryFnMemberAccessFact,
74    FactoryFnWholeObjectFact, FactoryPropertySource, FactoryReturnExport,
75    FactoryReturnObjectPropertyAccessFact, FactoryReturnObjectPropertySourceFact,
76    FactoryReturnObjectShapeExport, FlagPatterns, FluentChainMemberAccessFact,
77    FluentChainNewMemberAccessFact, ImportInfo, ImportedName, InstanceExportBindingFact,
78    LocalTypeDeclaration, MemberAccess, MemberInfo, MemberKind, ModuleInfo, ModuleLoadMechanism,
79    ParseResult, PlaywrightConditionalFixtureDefinitionFact, PlaywrightFixtureAliasFact,
80    PlaywrightFixtureDefinitionFact, PlaywrightFixtureTypeFact, PlaywrightFixtureUseFact,
81    PlaywrightTestBindingFact, PublicSignatureTypeReference, QualifiedClassMemberAccessFact,
82    ReExportInfo, RequireCallInfo, RequiredTypeMemberFact, SemanticFact, SourceParseDegradation,
83    SourceReadFailure, StringEnumMemberValueFact, TypeAliasSurfaceTargetFact, TypeMemberTypeEntry,
84    TypedPropertyMemberAccessFact, VisibilityTag, VitestModuleMockAction,
85    VitestModuleMockOperationFact, compute_line_offsets,
86};
87
88pub use astro::{
89    extract_astro_frontmatter, extract_astro_style_regions, extract_astro_template_regions,
90};
91pub use css::{
92    StylesheetTokens, ThemeScan, ThemeTokenDef, extract_apply_tokens, extract_apply_tokens_located,
93    extract_css_module_exports, extract_css_var_reads_located, scan_stylesheet_tokens,
94    scan_theme_blocks,
95};
96pub use css_classes::{
97    MarkupClassScan, MarkupClassToken, is_edit_distance_one, is_typo_edit, scan_markup_class_tokens,
98};
99pub use css_in_js::{
100    ConsumerQuery, CssInJsObjectSheets, CssInJsToken, CssInJsTokenDef, CssInJsTokenOrigin,
101    TokenConsumerHit, css_in_js_consumer_scan, css_in_js_object_sheets, css_in_js_theme_token_defs,
102    css_in_js_token_defs, css_in_js_virtual_stylesheet,
103};
104pub use css_metrics::{compute_css_analytics, parse_css_color_rgb};
105pub use glimmer::{is_glimmer_file, strip_glimmer_templates};
106pub use mdx::{extract_mdx_statements, extract_mdx_statements_mapped};
107pub use sfc::{
108    SourceRegion, extract_sfc_scripts, extract_sfc_styles, extract_sfc_template_regions,
109    is_sfc_file, sfc_merged_script_source,
110};
111pub use sfc_css::{
112    scoped_unused_classes, sfc_preprocessor_virtual_stylesheet, sfc_virtual_stylesheet,
113};
114pub use similar_code::extract_similar_code_functions;
115pub use source_map::{ExtractionResult, ScriptSection, embedded_script_sections};
116pub use tailwind::{TailwindArbitraryUse, scan_tailwind_arbitrary_values};
117
118#[expect(
119    clippy::expect_used,
120    reason = "static regex patterns are hard-coded analyzer invariants covered by extraction tests"
121)]
122fn static_regex(pattern: &str) -> regex::Regex {
123    regex::Regex::new(pattern).expect("static regex pattern should compile")
124}
125
126pub use parse::{parse_source_to_module, parse_source_to_module_with_flags};
127
128/// Leading UTF-8 byte order mark codepoint.
129///
130/// Windows editors (Notepad, older VS settings, some IDE plugins) emit a UTF-8
131/// BOM at the start of source files. fallow's contract is "UTF-8 with or
132/// without BOM; line offsets are computed against the post-BOM view; the BOM,
133/// if present on input, is preserved on output by `fallow fix`."
134const BOM_CHAR: char = '\u{FEFF}';
135// Small, cache-hot inputs are faster on one thread than through Rayon setup.
136// Larger file sets still use parallel parsing where parse work dominates.
137const PARALLEL_PARSE_FILE_THRESHOLD: usize = 32;
138
139/// Strip the leading UTF-8 BOM if present.
140///
141/// Called at every file-read entry point in this crate so the rest of the
142/// pipeline (content hash, `compute_line_offsets`, oxc parser, downstream
143/// analyses) sees a consistent post-BOM view. Mirrors the
144/// `fallow_config` layer (`config_writer.rs::BOM`) so config-shaped sources
145/// and source-code-shaped sources are processed symmetrically. See issue #475.
146#[must_use]
147fn strip_bom(source: &str) -> &str {
148    source.strip_prefix(BOM_CHAR).unwrap_or(source)
149}
150
151/// Parse all files, extracting imports and exports.
152///
153/// Small file sets use a sequential fast path to avoid parallel scheduling
154/// overhead; larger file sets use parallel extraction.
155/// Uses the cache to skip reparsing files whose content hasn't changed.
156///
157/// When `need_complexity` is true, per-function cyclomatic/cognitive complexity
158/// metrics are computed during parsing (needed by the `health` command).
159/// Pass `false` for dead-code analysis where complexity data is unused.
160///
161/// Flag detection uses the built-in patterns only. A caller with a resolved
162/// config uses [`parse_all_files_cancellable`] with the config's patterns,
163/// because the cache keys on them.
164pub fn parse_all_files(
165    files: &[DiscoveredFile],
166    cache: Option<&CacheStore>,
167    need_complexity: bool,
168) -> ParseResult {
169    parse_all_files_cancellable(
170        files,
171        cache,
172        need_complexity,
173        None,
174        &FlagPatterns::default(),
175    )
176}
177
178/// Parse all files, abandoning the remaining ones once `cancellation` is set.
179///
180/// Rayon's `map`/`collect` cannot short-circuit, so cancellation makes the
181/// per-file body a no-op instead of stopping the iteration: the scheduled
182/// items still drain, but at one atomic load each. The returned
183/// [`ParseResult`] is therefore truncated whenever the token flipped, and
184/// callers must treat a set token as a failed run rather than as a project
185/// with fewer modules.
186///
187/// `flag_patterns` are the user flag patterns that detection applies on top
188/// of the built-in ones. They must match the patterns the cache was keyed on.
189pub fn parse_all_files_cancellable(
190    files: &[DiscoveredFile],
191    cache: Option<&CacheStore>,
192    need_complexity: bool,
193    cancellation: Option<&AtomicBool>,
194    flag_patterns: &FlagPatterns,
195) -> ParseResult {
196    let parse_one = |file: &DiscoveredFile| {
197        if cancellation.is_some_and(|cancelled| cancelled.load(Ordering::SeqCst)) {
198            return ParseFileResult::default();
199        }
200        parse_single_file_cached(file, cache, need_complexity, flag_patterns)
201    };
202    let results: Vec<ParseFileResult> = if files.len() <= PARALLEL_PARSE_FILE_THRESHOLD {
203        files.iter().map(parse_one).collect()
204    } else {
205        files.par_iter().map(parse_one).collect()
206    };
207
208    let mut modules = Vec::with_capacity(results.len());
209    let mut read_failures = Vec::new();
210    let mut parse_degradations = Vec::new();
211    let mut hits = 0usize;
212    let mut misses = 0usize;
213    let mut parse_cpu_nanos = 0u64;
214    let mut files_read = 0u64;
215    let mut source_bytes_read = 0u64;
216    let mut css_masked_bytes = 0u64;
217
218    // `results` is a positional map over `files`, so zipping recovers the path
219    // for a module without carrying one on `ModuleInfo`.
220    for (file, result) in files.iter().zip(results) {
221        hits += result.cache_hits;
222        misses += result.cache_misses;
223        parse_cpu_nanos = parse_cpu_nanos.saturating_add(result.parse_cpu_nanos);
224        if let Some(bytes) = result.source_bytes_read {
225            files_read += 1;
226            source_bytes_read += bytes;
227        }
228        css_masked_bytes += result.css_masked_bytes;
229        if let Some(module) = result.module {
230            if module.parse_error_count > 0 {
231                parse_degradations.push(SourceParseDegradation {
232                    file_id: module.file_id,
233                    path: file.path.clone(),
234                    error_count: module.parse_error_count,
235                    panicked: module.parse_panicked,
236                });
237            }
238            modules.push(module);
239        }
240        if let Some(failure) = result.read_failure {
241            read_failures.push(failure);
242        }
243    }
244
245    if hits > 0 || misses > 0 {
246        tracing::info!(
247            cache_hits = hits,
248            cache_misses = misses,
249            "incremental cache stats"
250        );
251    }
252
253    ParseResult {
254        modules,
255        read_failures,
256        parse_degradations,
257        cache_hits: hits,
258        cache_misses: misses,
259        parse_cpu_ms: parse_cpu_nanos as f64 / 1_000_000.0,
260        files_read,
261        source_bytes_read,
262        css_masked_bytes,
263    }
264}
265
266#[derive(Default)]
267struct ParseFileResult {
268    module: Option<ModuleInfo>,
269    read_failure: Option<SourceReadFailure>,
270    cache_hits: usize,
271    cache_misses: usize,
272    parse_cpu_nanos: u64,
273    /// Source bytes read from disk for this file, or `None` when the file was
274    /// served from cache metadata without a read.
275    source_bytes_read: Option<u64>,
276    /// Source bytes that the CSS comment mask read during the parse.
277    css_masked_bytes: u64,
278}
279
280impl ParseFileResult {
281    fn cache_hit(module: ModuleInfo) -> Self {
282        Self {
283            module: Some(module),
284            read_failure: None,
285            cache_hits: 1,
286            cache_misses: 0,
287            parse_cpu_nanos: 0,
288            source_bytes_read: None,
289            css_masked_bytes: 0,
290        }
291    }
292
293    fn cache_miss(module: ModuleInfo, parse_cpu_nanos: u64) -> Self {
294        Self {
295            module: Some(module),
296            read_failure: None,
297            cache_hits: 0,
298            cache_misses: 1,
299            parse_cpu_nanos,
300            source_bytes_read: None,
301            css_masked_bytes: 0,
302        }
303    }
304
305    const fn with_source_bytes_read(mut self, bytes: usize) -> Self {
306        self.source_bytes_read = Some(bytes as u64);
307        self
308    }
309
310    fn read_failure(file: &DiscoveredFile, error: &std::io::Error) -> Self {
311        Self {
312            module: None,
313            read_failure: Some(SourceReadFailure {
314                file_id: file.id,
315                path: file.path.clone(),
316                error: error.to_string(),
317            }),
318            cache_hits: 0,
319            cache_misses: 0,
320            parse_cpu_nanos: 0,
321            source_bytes_read: None,
322            css_masked_bytes: 0,
323        }
324    }
325}
326
327/// Parse a single file, consulting the cache first.
328///
329/// Cache validation strategy (fast path -> slow path):
330/// 1. Open the file so unreadable sources cannot use stale cached analysis
331/// 2. Read mtime + ctime + size from the open handle
332/// 3. If all three match the cached entry -> cache hit, return immediately
333/// 4. Otherwise -> read file, compute content hash
334/// 5. If content hash matches cached entry -> cache hit (file was rewritten or
335///    `touch`ed but its content is unchanged)
336/// 6. Otherwise -> cache miss, full parse
337///
338/// Step 3 requires ctime as well as mtime because mtime is writer-controlled:
339/// a same-length rewrite whose mtime is restored (`touch -r`, a codemod, a
340/// `git checkout` of an equal-length revision) leaves `(mtime, size)`
341/// unchanged, and serving the cached module for it means reporting the OLD
342/// file's unused exports with an auto-fixable `remove-export` action. A file
343/// whose ctime moved falls through to step 4 and still hits on the content
344/// hash, so the cost of the stricter gate is one read, not a reparse.
345///
346/// Step 3 also needs a stored fingerprint that settled before the run that
347/// wrote it read the file. A same-length write in the same timestamp tick as
348/// the previous write keeps mtime, ctime and size, so the cache writer stores
349/// a recent fingerprint without its ctime
350/// (`SourceFingerprint::for_content_read_at`). Such an entry never matches at
351/// step 3, and the next run compares the content hash.
352fn parse_single_file_cached(
353    file: &DiscoveredFile,
354    cache: Option<&CacheStore>,
355    need_complexity: bool,
356    flag_patterns: &FlagPatterns,
357) -> ParseFileResult {
358    let cached_by_path = cache.and_then(|store| store.get_by_path_only(&file.path));
359
360    if let Some(cached) = cached_by_path
361        && cached.file_size == file.size_bytes
362    {
363        let source_file = match std::fs::File::open(&file.path) {
364            Ok(source_file) => source_file,
365            Err(error) => return ParseFileResult::read_failure(file, &error),
366        };
367        if let Ok(metadata) = source_file.metadata()
368            && metadata.len() == cached.file_size
369        {
370            let fingerprint =
371                fallow_types::source_fingerprint::SourceFingerprint::from_metadata(&metadata);
372            if cached.source_fingerprint() == fingerprint
373                && fingerprint.is_trustworthy_without_content()
374                && (!need_complexity || cached.complexity_extracted)
375            {
376                return ParseFileResult::cache_hit(cache::cached_to_module_opts(
377                    cached,
378                    file.id,
379                    need_complexity,
380                ));
381            }
382        }
383    }
384
385    let raw = match std::fs::read_to_string(&file.path) {
386        Ok(raw) => raw,
387        Err(error) => return ParseFileResult::read_failure(file, &error),
388    };
389    let source = strip_bom(&raw);
390    let content_hash = xxhash_rust::xxh3::xxh3_64(source.as_bytes());
391
392    if let Some(cached) = cached_by_path
393        && cached.content_hash == content_hash
394        && (!need_complexity || cached.complexity_extracted)
395    {
396        return ParseFileResult::cache_hit(cache::cached_to_module_opts(
397            cached,
398            file.id,
399            need_complexity,
400        ))
401        .with_source_bytes_read(raw.len());
402    }
403
404    let parse_start = std::time::Instant::now();
405    // Drop a count that a scan outside a parse left on this thread.
406    css::take_comment_masked_bytes();
407    let module = parse_source_to_module_with_flags(
408        file.id,
409        &file.path,
410        source,
411        content_hash,
412        need_complexity,
413        flag_patterns,
414    );
415    let parse_cpu_nanos = u64::try_from(parse_start.elapsed().as_nanos()).unwrap_or(u64::MAX);
416    let mut result =
417        ParseFileResult::cache_miss(module, parse_cpu_nanos).with_source_bytes_read(raw.len());
418    result.css_masked_bytes = css::take_comment_masked_bytes();
419    result
420}
421
422/// Parse a single file and extract module information (without complexity).
423#[must_use]
424pub fn parse_single_file(file: &DiscoveredFile) -> Option<ModuleInfo> {
425    let raw = std::fs::read_to_string(&file.path).ok()?;
426    let source = strip_bom(&raw);
427    let content_hash = xxhash_rust::xxh3::xxh3_64(source.as_bytes());
428    Some(parse_source_to_module(
429        file.id,
430        &file.path,
431        source,
432        content_hash,
433        false,
434    ))
435}
436
437/// Parse from in-memory content (for LSP, includes complexity).
438#[must_use]
439pub fn parse_from_content(file_id: FileId, path: &Path, content: &str) -> ModuleInfo {
440    let content = strip_bom(content);
441    let content_hash = xxhash_rust::xxh3::xxh3_64(content.as_bytes());
442    parse_source_to_module(file_id, path, content, content_hash, true)
443}
444
445#[cfg(all(test, not(miri)))]
446mod tests;
fallow_extract/lib.rs

fallow_extract/
lib.rs
