fallow_extract/
lib.rs

1//! Parsing and extraction engine for fallow codebase intelligence.
2//!
3//! This crate handles all file parsing: JS/TS via Oxc, Vue/Svelte SFC extraction,
4//! Astro frontmatter, MDX import/export extraction, CSS Module class name extraction,
5//! HTML asset reference extraction, and incremental caching of parse results.
6
7#![warn(missing_docs)]
8#![cfg_attr(not(test), deny(clippy::disallowed_methods))]
9#![cfg_attr(
10    test,
11    allow(
12        clippy::unwrap_used,
13        clippy::expect_used,
14        reason = "tests use unwrap and expect to keep fixture setup concise"
15    )
16)]
17
18mod asset_url;
19pub mod astro;
20mod binding_references;
21pub mod cache;
22pub(crate) mod complexity;
23mod component_contracts;
24pub mod css;
25pub mod css_classes;
26pub mod css_in_js;
27pub mod css_metrics;
28pub mod federation_runtime;
29pub mod flags;
30mod function_body;
31pub mod glimmer;
32pub(crate) mod graphql;
33pub(crate) mod html;
34pub(crate) mod iconify;
35pub mod inventory;
36mod jsdoc_attach;
37mod jsdoc_deprecated;
38pub mod mdx;
39mod module_info;
40pub mod og_image;
41mod parse;
42pub mod sfc;
43pub mod sfc_css;
44mod sfc_props;
45mod sfc_template;
46pub mod similar_code;
47mod source_map;
48pub mod suppress;
49/// Tailwind CSS arbitrary-value detection.
50pub mod tailwind;
51pub(crate) mod template_complexity;
52mod template_expression_scan;
53mod template_usage;
54/// Visitor utilities for AST extraction.
55pub mod visitor;
56
57use std::path::Path;
58use std::sync::atomic::{AtomicBool, Ordering};
59
60use rayon::prelude::*;
61
62use cache::CacheStore;
63use fallow_types::discover::{DiscoveredFile, FileId};
64
65pub use fallow_types::extract::{
66    AngularComponentFieldArrayTypeFact, AngularTemplateMemberAccessFact, AngularThisSpreadFact,
67    ClassHeritageInfo, ClassThisMemberAccessFact, ClassThisWholeObjectUseFact,
68    ComputedEnumKeyUseFact, DefaultImportWholeObjectUseFact, DynamicCustomElementRenderFact,
69    DynamicImportInfo, DynamicImportPattern, ExportInfo, ExportName,
70    ExportedObjectInstancePropertyFact, FactoryCallMemberAccessFact, FactoryFnMemberAccessFact,
71    FactoryFnWholeObjectFact, FactoryReturnExport, FactoryReturnObjectPropertyAccessFact,
72    FactoryReturnObjectShapeExport, FlagPatterns, FluentChainMemberAccessFact,
73    FluentChainNewMemberAccessFact, ImportInfo, ImportedName, InstanceExportBindingFact,
74    LocalTypeDeclaration, MemberAccess, MemberInfo, MemberKind, ModuleInfo, ModuleLoadMechanism,
75    ParseResult, PlaywrightFixtureAliasFact, PlaywrightFixtureDefinitionFact,
76    PlaywrightFixtureTypeFact, PlaywrightFixtureUseFact, PublicSignatureTypeReference,
77    QualifiedClassMemberAccessFact, ReExportInfo, RequireCallInfo, RequiredTypeMemberFact,
78    SemanticFact, SourceParseDegradation, SourceReadFailure, StringEnumMemberValueFact,
79    TypeAliasSurfaceTargetFact, TypeMemberTypeEntry, TypedPropertyMemberAccessFact, VisibilityTag,
80    VitestModuleMockAction, VitestModuleMockOperationFact, compute_line_offsets,
81};
82
83pub use astro::{
84    extract_astro_frontmatter, extract_astro_style_regions, extract_astro_template_regions,
85};
86pub use css::{
87    StylesheetTokens, ThemeScan, ThemeTokenDef, extract_apply_tokens, extract_apply_tokens_located,
88    extract_css_module_exports, extract_css_var_reads_located, scan_stylesheet_tokens,
89    scan_theme_blocks,
90};
91pub use css_classes::{
92    MarkupClassScan, MarkupClassToken, is_edit_distance_one, is_typo_edit, scan_markup_class_tokens,
93};
94pub use css_in_js::{
95    ConsumerQuery, CssInJsObjectSheets, CssInJsToken, CssInJsTokenDef, CssInJsTokenOrigin,
96    TokenConsumerHit, css_in_js_consumer_scan, css_in_js_object_sheets, css_in_js_theme_token_defs,
97    css_in_js_token_defs, css_in_js_virtual_stylesheet,
98};
99pub use css_metrics::{compute_css_analytics, parse_css_color_rgb};
100pub use glimmer::{is_glimmer_file, strip_glimmer_templates};
101pub use mdx::{extract_mdx_statements, extract_mdx_statements_mapped};
102pub use sfc::{
103    SourceRegion, extract_sfc_scripts, extract_sfc_styles, extract_sfc_template_regions,
104    is_sfc_file,
105};
106pub use sfc_css::{
107    scoped_unused_classes, sfc_preprocessor_virtual_stylesheet, sfc_virtual_stylesheet,
108};
109pub use similar_code::extract_similar_code_functions;
110pub use source_map::ExtractionResult;
111pub use tailwind::{TailwindArbitraryUse, scan_tailwind_arbitrary_values};
112
113#[expect(
114    clippy::expect_used,
115    reason = "static regex patterns are hard-coded analyzer invariants covered by extraction tests"
116)]
117fn static_regex(pattern: &str) -> regex::Regex {
118    regex::Regex::new(pattern).expect("static regex pattern should compile")
119}
120
121pub use parse::{parse_source_to_module, parse_source_to_module_with_flags};
122
123/// Leading UTF-8 byte order mark codepoint.
124///
125/// Windows editors (Notepad, older VS settings, some IDE plugins) emit a UTF-8
126/// BOM at the start of source files. fallow's contract is "UTF-8 with or
127/// without BOM; line offsets are computed against the post-BOM view; the BOM,
128/// if present on input, is preserved on output by `fallow fix`."
129const BOM_CHAR: char = '\u{FEFF}';
130// Small, cache-hot inputs are faster on one thread than through Rayon setup.
131// Larger file sets still use parallel parsing where parse work dominates.
132const PARALLEL_PARSE_FILE_THRESHOLD: usize = 32;
133
134/// Strip the leading UTF-8 BOM if present.
135///
136/// Called at every file-read entry point in this crate so the rest of the
137/// pipeline (content hash, `compute_line_offsets`, oxc parser, downstream
138/// analyses) sees a consistent post-BOM view. Mirrors the
139/// `fallow_config` layer (`config_writer.rs::BOM`) so config-shaped sources
140/// and source-code-shaped sources are processed symmetrically. See issue #475.
141#[must_use]
142fn strip_bom(source: &str) -> &str {
143    source.strip_prefix(BOM_CHAR).unwrap_or(source)
144}
145
146/// Parse all files, extracting imports and exports.
147///
148/// Small file sets use a sequential fast path to avoid parallel scheduling
149/// overhead; larger file sets use parallel extraction.
150/// Uses the cache to skip reparsing files whose content hasn't changed.
151///
152/// When `need_complexity` is true, per-function cyclomatic/cognitive complexity
153/// metrics are computed during parsing (needed by the `health` command).
154/// Pass `false` for dead-code analysis where complexity data is unused.
155///
156/// Flag detection uses the built-in patterns only. A caller with a resolved
157/// config uses [`parse_all_files_cancellable`] with the config's patterns,
158/// because the cache keys on them.
159pub fn parse_all_files(
160    files: &[DiscoveredFile],
161    cache: Option<&CacheStore>,
162    need_complexity: bool,
163) -> ParseResult {
164    parse_all_files_cancellable(
165        files,
166        cache,
167        need_complexity,
168        None,
169        &FlagPatterns::default(),
170    )
171}
172
173/// Parse all files, abandoning the remaining ones once `cancellation` is set.
174///
175/// Rayon's `map`/`collect` cannot short-circuit, so cancellation makes the
176/// per-file body a no-op instead of stopping the iteration: the scheduled
177/// items still drain, but at one atomic load each. The returned
178/// [`ParseResult`] is therefore truncated whenever the token flipped, and
179/// callers must treat a set token as a failed run rather than as a project
180/// with fewer modules.
181///
182/// `flag_patterns` are the user flag patterns that detection applies on top
183/// of the built-in ones. They must match the patterns the cache was keyed on.
184pub fn parse_all_files_cancellable(
185    files: &[DiscoveredFile],
186    cache: Option<&CacheStore>,
187    need_complexity: bool,
188    cancellation: Option<&AtomicBool>,
189    flag_patterns: &FlagPatterns,
190) -> ParseResult {
191    let parse_one = |file: &DiscoveredFile| {
192        if cancellation.is_some_and(|cancelled| cancelled.load(Ordering::SeqCst)) {
193            return ParseFileResult::default();
194        }
195        parse_single_file_cached(file, cache, need_complexity, flag_patterns)
196    };
197    let results: Vec<ParseFileResult> = if files.len() <= PARALLEL_PARSE_FILE_THRESHOLD {
198        files.iter().map(parse_one).collect()
199    } else {
200        files.par_iter().map(parse_one).collect()
201    };
202
203    let mut modules = Vec::with_capacity(results.len());
204    let mut read_failures = Vec::new();
205    let mut parse_degradations = Vec::new();
206    let mut hits = 0usize;
207    let mut misses = 0usize;
208    let mut parse_cpu_nanos = 0u64;
209    let mut files_read = 0u64;
210    let mut source_bytes_read = 0u64;
211    let mut css_masked_bytes = 0u64;
212
213    // `results` is a positional map over `files`, so zipping recovers the path
214    // for a module without carrying one on `ModuleInfo`.
215    for (file, result) in files.iter().zip(results) {
216        hits += result.cache_hits;
217        misses += result.cache_misses;
218        parse_cpu_nanos = parse_cpu_nanos.saturating_add(result.parse_cpu_nanos);
219        if let Some(bytes) = result.source_bytes_read {
220            files_read += 1;
221            source_bytes_read += bytes;
222        }
223        css_masked_bytes += result.css_masked_bytes;
224        if let Some(module) = result.module {
225            if module.parse_error_count > 0 {
226                parse_degradations.push(SourceParseDegradation {
227                    file_id: module.file_id,
228                    path: file.path.clone(),
229                    error_count: module.parse_error_count,
230                    panicked: module.parse_panicked,
231                });
232            }
233            modules.push(module);
234        }
235        if let Some(failure) = result.read_failure {
236            read_failures.push(failure);
237        }
238    }
239
240    if hits > 0 || misses > 0 {
241        tracing::info!(
242            cache_hits = hits,
243            cache_misses = misses,
244            "incremental cache stats"
245        );
246    }
247
248    ParseResult {
249        modules,
250        read_failures,
251        parse_degradations,
252        cache_hits: hits,
253        cache_misses: misses,
254        parse_cpu_ms: parse_cpu_nanos as f64 / 1_000_000.0,
255        files_read,
256        source_bytes_read,
257        css_masked_bytes,
258    }
259}
260
261#[derive(Default)]
262struct ParseFileResult {
263    module: Option<ModuleInfo>,
264    read_failure: Option<SourceReadFailure>,
265    cache_hits: usize,
266    cache_misses: usize,
267    parse_cpu_nanos: u64,
268    /// Source bytes read from disk for this file, or `None` when the file was
269    /// served from cache metadata without a read.
270    source_bytes_read: Option<u64>,
271    /// Source bytes that the CSS comment mask read during the parse.
272    css_masked_bytes: u64,
273}
274
275impl ParseFileResult {
276    fn cache_hit(module: ModuleInfo) -> Self {
277        Self {
278            module: Some(module),
279            read_failure: None,
280            cache_hits: 1,
281            cache_misses: 0,
282            parse_cpu_nanos: 0,
283            source_bytes_read: None,
284            css_masked_bytes: 0,
285        }
286    }
287
288    fn cache_miss(module: ModuleInfo, parse_cpu_nanos: u64) -> Self {
289        Self {
290            module: Some(module),
291            read_failure: None,
292            cache_hits: 0,
293            cache_misses: 1,
294            parse_cpu_nanos,
295            source_bytes_read: None,
296            css_masked_bytes: 0,
297        }
298    }
299
300    const fn with_source_bytes_read(mut self, bytes: usize) -> Self {
301        self.source_bytes_read = Some(bytes as u64);
302        self
303    }
304
305    fn read_failure(file: &DiscoveredFile, error: &std::io::Error) -> Self {
306        Self {
307            module: None,
308            read_failure: Some(SourceReadFailure {
309                file_id: file.id,
310                path: file.path.clone(),
311                error: error.to_string(),
312            }),
313            cache_hits: 0,
314            cache_misses: 0,
315            parse_cpu_nanos: 0,
316            source_bytes_read: None,
317            css_masked_bytes: 0,
318        }
319    }
320}
321
322/// Parse a single file, consulting the cache first.
323///
324/// Cache validation strategy (fast path -> slow path):
325/// 1. Open the file so unreadable sources cannot use stale cached analysis
326/// 2. Read mtime + ctime + size from the open handle
327/// 3. If all three match the cached entry -> cache hit, return immediately
328/// 4. Otherwise -> read file, compute content hash
329/// 5. If content hash matches cached entry -> cache hit (file was rewritten or
330///    `touch`ed but its content is unchanged)
331/// 6. Otherwise -> cache miss, full parse
332///
333/// Step 3 requires ctime as well as mtime because mtime is writer-controlled:
334/// a same-length rewrite whose mtime is restored (`touch -r`, a codemod, a
335/// `git checkout` of an equal-length revision) leaves `(mtime, size)`
336/// unchanged, and serving the cached module for it means reporting the OLD
337/// file's unused exports with an auto-fixable `remove-export` action. A file
338/// whose ctime moved falls through to step 4 and still hits on the content
339/// hash, so the cost of the stricter gate is one read, not a reparse.
340///
341/// Step 3 also needs a stored fingerprint that settled before the run that
342/// wrote it read the file. A same-length write in the same timestamp tick as
343/// the previous write keeps mtime, ctime and size, so the cache writer stores
344/// a recent fingerprint without its ctime
345/// (`SourceFingerprint::for_content_read_at`). Such an entry never matches at
346/// step 3, and the next run compares the content hash.
347fn parse_single_file_cached(
348    file: &DiscoveredFile,
349    cache: Option<&CacheStore>,
350    need_complexity: bool,
351    flag_patterns: &FlagPatterns,
352) -> ParseFileResult {
353    let cached_by_path = cache.and_then(|store| store.get_by_path_only(&file.path));
354
355    if let Some(cached) = cached_by_path
356        && cached.file_size == file.size_bytes
357    {
358        let source_file = match std::fs::File::open(&file.path) {
359            Ok(source_file) => source_file,
360            Err(error) => return ParseFileResult::read_failure(file, &error),
361        };
362        if let Ok(metadata) = source_file.metadata()
363            && metadata.len() == cached.file_size
364        {
365            let fingerprint =
366                fallow_types::source_fingerprint::SourceFingerprint::from_metadata(&metadata);
367            if cached.source_fingerprint() == fingerprint
368                && fingerprint.is_trustworthy_without_content()
369                && (!need_complexity || cached.complexity_extracted)
370            {
371                return ParseFileResult::cache_hit(cache::cached_to_module_opts(
372                    cached,
373                    file.id,
374                    need_complexity,
375                ));
376            }
377        }
378    }
379
380    let raw = match std::fs::read_to_string(&file.path) {
381        Ok(raw) => raw,
382        Err(error) => return ParseFileResult::read_failure(file, &error),
383    };
384    let source = strip_bom(&raw);
385    let content_hash = xxhash_rust::xxh3::xxh3_64(source.as_bytes());
386
387    if let Some(cached) = cached_by_path
388        && cached.content_hash == content_hash
389        && (!need_complexity || cached.complexity_extracted)
390    {
391        return ParseFileResult::cache_hit(cache::cached_to_module_opts(
392            cached,
393            file.id,
394            need_complexity,
395        ))
396        .with_source_bytes_read(raw.len());
397    }
398
399    let parse_start = std::time::Instant::now();
400    // Drop a count that a scan outside a parse left on this thread.
401    css::take_comment_masked_bytes();
402    let module = parse_source_to_module_with_flags(
403        file.id,
404        &file.path,
405        source,
406        content_hash,
407        need_complexity,
408        flag_patterns,
409    );
410    let parse_cpu_nanos = u64::try_from(parse_start.elapsed().as_nanos()).unwrap_or(u64::MAX);
411    let mut result =
412        ParseFileResult::cache_miss(module, parse_cpu_nanos).with_source_bytes_read(raw.len());
413    result.css_masked_bytes = css::take_comment_masked_bytes();
414    result
415}
416
417/// Parse a single file and extract module information (without complexity).
418#[must_use]
419pub fn parse_single_file(file: &DiscoveredFile) -> Option<ModuleInfo> {
420    let raw = std::fs::read_to_string(&file.path).ok()?;
421    let source = strip_bom(&raw);
422    let content_hash = xxhash_rust::xxh3::xxh3_64(source.as_bytes());
423    Some(parse_source_to_module(
424        file.id,
425        &file.path,
426        source,
427        content_hash,
428        false,
429    ))
430}
431
432/// Parse from in-memory content (for LSP, includes complexity).
433#[must_use]
434pub fn parse_from_content(file_id: FileId, path: &Path, content: &str) -> ModuleInfo {
435    let content = strip_bom(content);
436    let content_hash = xxhash_rust::xxh3::xxh3_64(content.as_bytes());
437    parse_source_to_module(file_id, path, content, content_hash, true)
438}
439
440#[cfg(all(test, not(miri)))]
441mod tests;
fallow_extract/lib.rs

fallow_extract/
lib.rs
