fallow_extract/
lib.rs

1//! Parsing and extraction engine for fallow codebase intelligence.
2//!
3//! This crate handles all file parsing: JS/TS via Oxc, Vue/Svelte SFC extraction,
4//! Astro frontmatter, MDX import/export extraction, CSS Module class name extraction,
5//! HTML asset reference extraction, and incremental caching of parse results.
6
7#![warn(missing_docs)]
8#![cfg_attr(not(test), deny(clippy::disallowed_methods))]
9#![cfg_attr(
10    test,
11    allow(
12        clippy::unwrap_used,
13        clippy::expect_used,
14        reason = "tests use unwrap and expect to keep fixture setup concise"
15    )
16)]
17
18pub mod alchemy;
19mod asset_url;
20pub mod astro;
21mod binding_references;
22pub mod cache;
23pub(crate) mod complexity;
24mod complexity_names;
25mod component_contracts;
26pub(crate) mod cost;
27pub mod css;
28pub mod css_classes;
29pub mod css_in_js;
30mod css_mask;
31mod css_masked_bytes;
32pub mod css_metrics;
33mod css_module_classes;
34pub mod dom_classes;
35pub mod effect_facts;
36pub mod federation_runtime;
37pub mod flags;
38mod function_body;
39pub(crate) mod function_hash;
40pub mod glimmer;
41pub(crate) mod graphql;
42pub(crate) mod html;
43pub(crate) mod iconify;
44pub mod inventory;
45pub mod js_text;
46mod jsdoc_attach;
47mod jsdoc_deprecated;
48mod literal_membership;
49pub mod mdx;
50mod module_info;
51mod module_parse;
52mod nesting;
53pub mod og_image;
54mod parse;
55mod parse_degradation;
56pub mod sfc;
57pub mod sfc_css;
58mod sfc_props;
59mod sfc_template;
60pub mod similar_code;
61mod source_map;
62mod stack;
63pub mod suppress;
64/// Tailwind CSS arbitrary-value detection.
65pub mod tailwind;
66pub(crate) mod template_complexity;
67mod template_expression_scan;
68mod template_usage;
69/// Visitor utilities for AST extraction.
70pub mod visitor;
71
72use std::path::Path;
73use std::sync::atomic::{AtomicBool, Ordering};
74
75use rayon::prelude::*;
76
77use cache::CacheStore;
78use fallow_types::discover::{DiscoveredFile, FileId};
79
80pub use fallow_types::extract::{
81    AngularComponentFieldArrayTypeFact, AngularTemplateMemberAccessFact, AngularThisSpreadFact,
82    ClassHeritageInfo, ClassThisMemberAccessFact, ClassThisWholeObjectUseFact,
83    ComputedEnumKeyUseFact, DefaultImportWholeObjectUseFact, DynamicCustomElementRenderFact,
84    DynamicImportInfo, DynamicImportPattern, ExportInfo, ExportName,
85    ExportedObjectInstancePropertyFact, FactoryCallMemberAccessFact, FactoryFnMemberAccessFact,
86    FactoryFnWholeObjectFact, FactoryPropertySource, FactoryReturnExport,
87    FactoryReturnObjectPropertyAccessFact, FactoryReturnObjectPropertySourceFact,
88    FactoryReturnObjectShapeExport, FlagPatterns, FluentChainMemberAccessFact,
89    FluentChainNewMemberAccessFact, ImportInfo, ImportedName, InstanceExportBindingFact,
90    LocalTypeDeclaration, MemberAccess, MemberInfo, MemberKind, ModuleInfo, ModuleLoadMechanism,
91    ParseResult, PlaywrightConditionalFixtureDefinitionFact, PlaywrightFixtureAliasFact,
92    PlaywrightFixtureDefinitionFact, PlaywrightFixtureTypeFact, PlaywrightFixtureUseFact,
93    PlaywrightTestBindingFact, PublicSignatureTypeReference, QualifiedClassMemberAccessFact,
94    ReExportInfo, RequireCallInfo, RequiredTypeMemberFact, SemanticFact, SourceParseDegradation,
95    SourceReadFailure, StringEnumMemberValueFact, TypeAliasSurfaceTargetFact, TypeMemberTypeEntry,
96    TypedPropertyMemberAccessFact, VisibilityTag, VitestModuleMockAction,
97    VitestModuleMockOperationFact, compute_line_offsets,
98};
99
100pub use astro::{
101    extract_astro_frontmatter, extract_astro_style_regions, extract_astro_template_regions,
102};
103pub use css::{
104    StylesheetTokens, ThemeScan, ThemeTokenDef, extract_apply_tokens, extract_apply_tokens_located,
105    extract_css_module_exports, extract_css_var_reads_located, scan_stylesheet_tokens,
106    scan_theme_blocks,
107};
108pub use css_classes::{
109    MarkupClassScan, MarkupClassToken, is_edit_distance_one, is_typo_edit, scan_markup_class_tokens,
110};
111pub use css_in_js::{
112    ConsumerQuery, CssInJsObjectSheets, CssInJsToken, CssInJsTokenDef, CssInJsTokenOrigin,
113    TokenConsumerHit, css_in_js_consumer_scan, css_in_js_object_sheets, css_in_js_theme_token_defs,
114    css_in_js_token_defs, css_in_js_virtual_stylesheet,
115};
116pub use css_metrics::{compute_css_analytics, parse_css_color_rgb};
117pub use dom_classes::{DomClassScan, scan_dom_class_values};
118pub use glimmer::{is_glimmer_file, strip_glimmer_templates};
119pub use js_text::JsText;
120pub use mdx::{extract_mdx_statements, extract_mdx_statements_mapped};
121pub use nesting::{
122    MAX_NESTING_DEPTH, exceeds_nesting_limit, parse_bounded, parse_bounded_or_default,
123    parse_style_bounded,
124};
125pub use sfc::{
126    SourceRegion, extract_sfc_scripts, extract_sfc_styles, extract_sfc_template_regions,
127    is_sfc_file, sfc_merged_script_source,
128};
129pub use sfc_css::{
130    scoped_unused_classes, sfc_preprocessor_virtual_stylesheet, sfc_virtual_stylesheet,
131};
132pub use similar_code::extract_similar_code_functions;
133pub use source_map::{ExtractionResult, ScriptSection, embedded_script_sections};
134pub use stack::register_thread_stack;
135pub use tailwind::{TailwindArbitraryUse, scan_tailwind_arbitrary_values};
136
137#[expect(
138    clippy::expect_used,
139    reason = "static regex patterns are hard-coded analyzer invariants covered by extraction tests"
140)]
141fn static_regex(pattern: &str) -> regex::Regex {
142    regex::Regex::new(pattern).expect("static regex pattern should compile")
143}
144
145pub use module_parse::{parse_source_to_module, parse_source_to_module_with_flags};
146
147/// Leading UTF-8 byte order mark codepoint.
148///
149/// Windows editors (Notepad, older VS settings, some IDE plugins) emit a UTF-8
150/// BOM at the start of source files. fallow's contract is "UTF-8 with or
151/// without BOM; line offsets are computed against the post-BOM view; the BOM,
152/// if present on input, is preserved on output by `fallow fix`."
153const BOM_CHAR: char = '\u{FEFF}';
154// Small, cache-hot inputs are faster on one thread than through Rayon setup.
155// Larger file sets still use parallel parsing where parse work dominates.
156const PARALLEL_PARSE_FILE_THRESHOLD: usize = 32;
157
158/// Strip the leading UTF-8 BOM if present.
159///
160/// Called at every file-read entry point in this crate so the rest of the
161/// pipeline (content hash, `compute_line_offsets`, oxc parser, downstream
162/// analyses) sees a consistent post-BOM view. Mirrors the
163/// `fallow_config` layer (`config_writer.rs::BOM`) so config-shaped sources
164/// and source-code-shaped sources are processed symmetrically. See issue #475.
165#[must_use]
166fn strip_bom(source: &str) -> &str {
167    source.strip_prefix(BOM_CHAR).unwrap_or(source)
168}
169
170/// Parse all files, extracting imports and exports.
171///
172/// Small file sets use a sequential fast path to avoid parallel scheduling
173/// overhead; larger file sets use parallel extraction.
174/// Uses the cache to skip reparsing files whose content hasn't changed.
175///
176/// When `need_complexity` is true, per-function cyclomatic/cognitive complexity
177/// metrics are computed during parsing (needed by the `health` command).
178/// Pass `false` for dead-code analysis where complexity data is unused.
179///
180/// Flag detection uses the built-in patterns only. A caller with a resolved
181/// config uses [`parse_all_files_cancellable`] with the config's patterns,
182/// because the cache keys on them.
183pub fn parse_all_files(
184    files: &[DiscoveredFile],
185    cache: Option<&CacheStore>,
186    need_complexity: bool,
187) -> ParseResult {
188    parse_all_files_cancellable(
189        files,
190        cache,
191        need_complexity,
192        None,
193        &FlagPatterns::default(),
194    )
195}
196
197/// Parse all files, abandoning the remaining ones once `cancellation` is set.
198///
199/// Rayon's `map`/`collect` cannot short-circuit, so cancellation makes the
200/// per-file body a no-op instead of stopping the iteration: the scheduled
201/// items still drain, but at one atomic load each. The returned
202/// [`ParseResult`] is therefore truncated whenever the token flipped, and
203/// callers must treat a set token as a failed run rather than as a project
204/// with fewer modules.
205///
206/// `flag_patterns` are the user flag patterns that detection applies on top
207/// of the built-in ones. They must match the patterns the cache was keyed on.
208pub fn parse_all_files_cancellable(
209    files: &[DiscoveredFile],
210    cache: Option<&CacheStore>,
211    need_complexity: bool,
212    cancellation: Option<&AtomicBool>,
213    flag_patterns: &FlagPatterns,
214) -> ParseResult {
215    let parse_one = |file: &DiscoveredFile| {
216        if cancellation.is_some_and(|cancelled| cancelled.load(Ordering::SeqCst)) {
217            return ParseFileResult::default();
218        }
219        parse_single_file_cached(file, cache, need_complexity, flag_patterns)
220    };
221    let results: Vec<ParseFileResult> = if files.len() <= PARALLEL_PARSE_FILE_THRESHOLD {
222        files.iter().map(parse_one).collect()
223    } else {
224        files.par_iter().map(parse_one).collect()
225    };
226
227    let mut modules = Vec::with_capacity(results.len());
228    let mut read_failures = Vec::new();
229    let mut parse_degradations = Vec::new();
230    let mut hits = 0usize;
231    let mut misses = 0usize;
232    let mut parse_cpu_nanos = 0u64;
233    let mut files_read = 0u64;
234    let mut source_bytes_read = 0u64;
235    let mut css_masked_bytes = 0u64;
236
237    // `results` is a positional map over `files`, so zipping recovers the path
238    // for a module without carrying one on `ModuleInfo`.
239    for (file, result) in files.iter().zip(results) {
240        hits += result.cache_hits;
241        misses += result.cache_misses;
242        parse_cpu_nanos = parse_cpu_nanos.saturating_add(result.parse_cpu_nanos);
243        if let Some(bytes) = result.source_bytes_read {
244            files_read += 1;
245            source_bytes_read += bytes;
246        }
247        css_masked_bytes += result.css_masked_bytes;
248        if let Some(module) = result.module {
249            if module.parse_error_count > 0 {
250                parse_degradations.push(SourceParseDegradation {
251                    file_id: module.file_id,
252                    path: file.path.clone(),
253                    error_count: module.parse_error_count,
254                    panicked: module.parse_panicked,
255                    nesting_limit: module.parse_nesting_too_deep.then_some(MAX_NESTING_DEPTH),
256                });
257            }
258            modules.push(module);
259        }
260        if let Some(failure) = result.read_failure {
261            read_failures.push(failure);
262        }
263    }
264
265    if hits > 0 || misses > 0 {
266        tracing::info!(
267            cache_hits = hits,
268            cache_misses = misses,
269            "incremental cache stats"
270        );
271    }
272
273    ParseResult {
274        modules,
275        read_failures,
276        parse_degradations,
277        cache_hits: hits,
278        cache_misses: misses,
279        parse_cpu_ms: parse_cpu_nanos as f64 / 1_000_000.0,
280        files_read,
281        source_bytes_read,
282        css_masked_bytes,
283    }
284}
285
286#[derive(Default)]
287struct ParseFileResult {
288    module: Option<ModuleInfo>,
289    read_failure: Option<SourceReadFailure>,
290    cache_hits: usize,
291    cache_misses: usize,
292    parse_cpu_nanos: u64,
293    /// Source bytes read from disk for this file, or `None` when the file was
294    /// served from cache metadata without a read.
295    source_bytes_read: Option<u64>,
296    /// Source bytes that the CSS comment mask read during the parse.
297    css_masked_bytes: u64,
298}
299
300impl ParseFileResult {
301    fn cache_hit(module: ModuleInfo) -> Self {
302        Self {
303            module: Some(module),
304            read_failure: None,
305            cache_hits: 1,
306            cache_misses: 0,
307            parse_cpu_nanos: 0,
308            source_bytes_read: None,
309            css_masked_bytes: 0,
310        }
311    }
312
313    fn cache_miss(module: ModuleInfo, parse_cpu_nanos: u64) -> Self {
314        Self {
315            module: Some(module),
316            read_failure: None,
317            cache_hits: 0,
318            cache_misses: 1,
319            parse_cpu_nanos,
320            source_bytes_read: None,
321            css_masked_bytes: 0,
322        }
323    }
324
325    const fn with_source_bytes_read(mut self, bytes: usize) -> Self {
326        self.source_bytes_read = Some(bytes as u64);
327        self
328    }
329
330    fn read_failure(file: &DiscoveredFile, error: &std::io::Error) -> Self {
331        Self {
332            module: None,
333            read_failure: Some(SourceReadFailure {
334                file_id: file.id,
335                path: file.path.clone(),
336                error: error.to_string(),
337            }),
338            cache_hits: 0,
339            cache_misses: 0,
340            parse_cpu_nanos: 0,
341            source_bytes_read: None,
342            css_masked_bytes: 0,
343        }
344    }
345}
346
347/// Parse a single file, consulting the cache first.
348///
349/// Cache validation strategy (fast path -> slow path):
350/// 1. Open the file so unreadable sources cannot use stale cached analysis
351/// 2. Read mtime + ctime + size from the open handle
352/// 3. If all three match the cached entry -> cache hit, return immediately
353/// 4. Otherwise -> read file, compute content hash
354/// 5. If content hash matches cached entry -> cache hit (file was rewritten or
355///    `touch`ed but its content is unchanged)
356/// 6. Otherwise -> cache miss, full parse
357///
358/// Step 3 requires ctime as well as mtime because mtime is writer-controlled:
359/// a same-length rewrite whose mtime is restored (`touch -r`, a codemod, a
360/// `git checkout` of an equal-length revision) leaves `(mtime, size)`
361/// unchanged, and serving the cached module for it means reporting the OLD
362/// file's unused exports with an auto-fixable `remove-export` action. A file
363/// whose ctime moved falls through to step 4 and still hits on the content
364/// hash, so the cost of the stricter gate is one read, not a reparse.
365///
366/// Step 3 also needs a stored fingerprint that settled before the run that
367/// wrote it read the file. A same-length write in the same timestamp tick as
368/// the previous write keeps mtime, ctime and size, so the cache writer stores
369/// a recent fingerprint without its ctime
370/// (`SourceFingerprint::for_content_read_at`). Such an entry never matches at
371/// step 3, and the next run compares the content hash.
372fn parse_single_file_cached(
373    file: &DiscoveredFile,
374    cache: Option<&CacheStore>,
375    need_complexity: bool,
376    flag_patterns: &FlagPatterns,
377) -> ParseFileResult {
378    let cached_by_path = cache.and_then(|store| store.get_by_path_only(&file.path));
379
380    if let Some(cached) = cached_by_path
381        && cached.file_size == file.size_bytes
382    {
383        let source_file = match std::fs::File::open(&file.path) {
384            Ok(source_file) => source_file,
385            Err(error) => return ParseFileResult::read_failure(file, &error),
386        };
387        if let Ok(metadata) = source_file.metadata()
388            && metadata.len() == cached.file_size
389        {
390            let fingerprint =
391                fallow_types::source_fingerprint::SourceFingerprint::from_metadata(&metadata);
392            if cached.source_fingerprint() == fingerprint
393                && fingerprint.is_trustworthy_without_content()
394                && (!need_complexity || cached.complexity_extracted)
395            {
396                return ParseFileResult::cache_hit(cache::cached_to_module_opts(
397                    cached,
398                    file.id,
399                    need_complexity,
400                ));
401            }
402        }
403    }
404
405    let raw = match std::fs::read_to_string(&file.path) {
406        Ok(raw) => raw,
407        Err(error) => return ParseFileResult::read_failure(file, &error),
408    };
409    let source = strip_bom(&raw);
410    let content_hash = xxhash_rust::xxh3::xxh3_64(source.as_bytes());
411
412    if let Some(cached) = cached_by_path
413        && cached.content_hash == content_hash
414        && (!need_complexity || cached.complexity_extracted)
415    {
416        return ParseFileResult::cache_hit(cache::cached_to_module_opts(
417            cached,
418            file.id,
419            need_complexity,
420        ))
421        .with_source_bytes_read(raw.len());
422    }
423
424    let parse_start = std::time::Instant::now();
425    // Drop a count that a scan outside a parse left on this thread.
426    css_masked_bytes::take();
427    let module = parse_source_to_module_with_flags(
428        file.id,
429        &file.path,
430        source,
431        content_hash,
432        need_complexity,
433        flag_patterns,
434    );
435    let parse_cpu_nanos = u64::try_from(parse_start.elapsed().as_nanos()).unwrap_or(u64::MAX);
436    let mut result =
437        ParseFileResult::cache_miss(module, parse_cpu_nanos).with_source_bytes_read(raw.len());
438    result.css_masked_bytes = css_masked_bytes::take();
439    result
440}
441
442/// Parse a single file and extract module information (without complexity).
443#[must_use]
444pub fn parse_single_file(file: &DiscoveredFile) -> Option<ModuleInfo> {
445    let raw = std::fs::read_to_string(&file.path).ok()?;
446    let source = strip_bom(&raw);
447    let content_hash = xxhash_rust::xxh3::xxh3_64(source.as_bytes());
448    Some(parse_source_to_module(
449        file.id,
450        &file.path,
451        source,
452        content_hash,
453        false,
454    ))
455}
456
457/// Parse from in-memory content (for LSP, includes complexity).
458#[must_use]
459pub fn parse_from_content(file_id: FileId, path: &Path, content: &str) -> ModuleInfo {
460    let content = strip_bom(content);
461    let content_hash = xxhash_rust::xxh3::xxh3_64(content.as_bytes());
462    parse_source_to_module(file_id, path, content, content_hash, true)
463}
464
465#[cfg(all(test, not(miri)))]
466mod tests;
fallow_extract/lib.rs

fallow_extract/
lib.rs
