fallow_extract/
lib.rs

1//! Parsing and extraction engine for fallow codebase intelligence.
2//!
3//! This crate handles all file parsing: JS/TS via Oxc, Vue/Svelte SFC extraction,
4//! Astro frontmatter, MDX import/export extraction, CSS Module class name extraction,
5//! HTML asset reference extraction, and incremental caching of parse results.
6
7#![warn(missing_docs)]
8#![cfg_attr(not(test), deny(clippy::disallowed_methods))]
9#![cfg_attr(
10    test,
11    allow(
12        clippy::unwrap_used,
13        clippy::expect_used,
14        reason = "tests use unwrap and expect to keep fixture setup concise"
15    )
16)]
17
18mod asset_url;
19pub mod astro;
20pub mod cache;
21pub(crate) mod complexity;
22pub mod css;
23pub mod css_classes;
24pub mod css_in_js;
25pub mod css_metrics;
26pub mod federation_runtime;
27pub mod flags;
28pub mod glimmer;
29pub(crate) mod graphql;
30pub(crate) mod html;
31pub(crate) mod iconify;
32pub mod inventory;
33mod jsdoc_attach;
34mod jsdoc_deprecated;
35pub mod mdx;
36mod module_info;
37pub mod og_image;
38mod parse;
39pub mod sfc;
40pub mod sfc_css;
41mod sfc_props;
42mod sfc_template;
43pub mod similar_code;
44mod source_map;
45pub mod suppress;
46/// Tailwind CSS arbitrary-value detection.
47pub mod tailwind;
48pub(crate) mod template_complexity;
49mod template_expression_scan;
50mod template_usage;
51/// Visitor utilities for AST extraction.
52pub mod visitor;
53
54use std::path::Path;
55use std::sync::atomic::{AtomicBool, Ordering};
56
57use rayon::prelude::*;
58
59use cache::CacheStore;
60use fallow_types::discover::{DiscoveredFile, FileId};
61
62pub use fallow_types::extract::{
63    AngularComponentFieldArrayTypeFact, AngularTemplateMemberAccessFact, AngularThisSpreadFact,
64    ClassHeritageInfo, ClassThisMemberAccessFact, ClassThisWholeObjectUseFact,
65    ComputedEnumKeyUseFact, DefaultImportWholeObjectUseFact, DynamicCustomElementRenderFact,
66    DynamicImportInfo, DynamicImportPattern, ExportInfo, ExportName,
67    ExportedObjectInstancePropertyFact, FactoryCallMemberAccessFact, FactoryFnMemberAccessFact,
68    FactoryFnWholeObjectFact, FactoryReturnExport, FactoryReturnObjectPropertyAccessFact,
69    FactoryReturnObjectShapeExport, FluentChainMemberAccessFact, FluentChainNewMemberAccessFact,
70    ImportInfo, ImportedName, InstanceExportBindingFact, LocalTypeDeclaration, MemberAccess,
71    MemberInfo, MemberKind, ModuleInfo, ModuleLoadMechanism, ParseResult,
72    PlaywrightFixtureAliasFact, PlaywrightFixtureDefinitionFact, PlaywrightFixtureTypeFact,
73    PlaywrightFixtureUseFact, PublicSignatureTypeReference, QualifiedClassMemberAccessFact,
74    ReExportInfo, RequireCallInfo, RequiredTypeMemberFact, SemanticFact, SourceParseDegradation,
75    SourceReadFailure, StringEnumMemberValueFact, TypeAliasSurfaceTargetFact, TypeMemberTypeEntry,
76    TypedPropertyMemberAccessFact, VisibilityTag, VitestModuleMockAction,
77    VitestModuleMockOperationFact, compute_line_offsets,
78};
79
80pub use astro::{
81    extract_astro_frontmatter, extract_astro_style_regions, extract_astro_template_regions,
82};
83pub use css::{
84    ThemeScan, ThemeTokenDef, extract_apply_tokens, extract_apply_tokens_located,
85    extract_css_module_exports, extract_css_var_reads_located, scan_theme_blocks,
86};
87pub use css_classes::{
88    MarkupClassScan, MarkupClassToken, is_edit_distance_one, is_typo_edit, scan_markup_class_tokens,
89};
90pub use css_in_js::{
91    ConsumerQuery, CssInJsObjectSheets, CssInJsToken, CssInJsTokenDef, CssInJsTokenOrigin,
92    TokenConsumerHit, css_in_js_consumer_scan, css_in_js_object_sheets, css_in_js_theme_token_defs,
93    css_in_js_token_defs, css_in_js_virtual_stylesheet,
94};
95pub use css_metrics::{compute_css_analytics, parse_css_color_rgb};
96pub use glimmer::{is_glimmer_file, strip_glimmer_templates};
97pub use mdx::{extract_mdx_statements, extract_mdx_statements_mapped};
98pub use sfc::{
99    SourceRegion, extract_sfc_scripts, extract_sfc_styles, extract_sfc_template_regions,
100    is_sfc_file,
101};
102pub use sfc_css::{
103    scoped_unused_classes, sfc_preprocessor_virtual_stylesheet, sfc_virtual_stylesheet,
104};
105pub use similar_code::extract_similar_code_functions;
106pub use source_map::ExtractionResult;
107pub use tailwind::{TailwindArbitraryUse, scan_tailwind_arbitrary_values};
108
109#[expect(
110    clippy::expect_used,
111    reason = "static regex patterns are hard-coded analyzer invariants covered by extraction tests"
112)]
113fn static_regex(pattern: &str) -> regex::Regex {
114    regex::Regex::new(pattern).expect("static regex pattern should compile")
115}
116
117pub use parse::parse_source_to_module;
118
119/// Leading UTF-8 byte order mark codepoint.
120///
121/// Windows editors (Notepad, older VS settings, some IDE plugins) emit a UTF-8
122/// BOM at the start of source files. fallow's contract is "UTF-8 with or
123/// without BOM; line offsets are computed against the post-BOM view; the BOM,
124/// if present on input, is preserved on output by `fallow fix`."
125const BOM_CHAR: char = '\u{FEFF}';
126// Small, cache-hot inputs are faster on one thread than through Rayon setup.
127// Larger file sets still use parallel parsing where parse work dominates.
128const PARALLEL_PARSE_FILE_THRESHOLD: usize = 32;
129
130/// Strip the leading UTF-8 BOM if present.
131///
132/// Called at every file-read entry point in this crate so the rest of the
133/// pipeline (content hash, `compute_line_offsets`, oxc parser, downstream
134/// analyses) sees a consistent post-BOM view. Mirrors the
135/// `fallow_config` layer (`config_writer.rs::BOM`) so config-shaped sources
136/// and source-code-shaped sources are processed symmetrically. See issue #475.
137#[must_use]
138fn strip_bom(source: &str) -> &str {
139    source.strip_prefix(BOM_CHAR).unwrap_or(source)
140}
141
142/// Parse all files, extracting imports and exports.
143///
144/// Small file sets use a sequential fast path to avoid parallel scheduling
145/// overhead; larger file sets use parallel extraction.
146/// Uses the cache to skip reparsing files whose content hasn't changed.
147///
148/// When `need_complexity` is true, per-function cyclomatic/cognitive complexity
149/// metrics are computed during parsing (needed by the `health` command).
150/// Pass `false` for dead-code analysis where complexity data is unused.
151pub fn parse_all_files(
152    files: &[DiscoveredFile],
153    cache: Option<&CacheStore>,
154    need_complexity: bool,
155) -> ParseResult {
156    parse_all_files_cancellable(files, cache, need_complexity, None)
157}
158
159/// Parse all files, abandoning the remaining ones once `cancellation` is set.
160///
161/// Rayon's `map`/`collect` cannot short-circuit, so cancellation makes the
162/// per-file body a no-op instead of stopping the iteration: the scheduled
163/// items still drain, but at one atomic load each. The returned
164/// [`ParseResult`] is therefore truncated whenever the token flipped, and
165/// callers must treat a set token as a failed run rather than as a project
166/// with fewer modules.
167pub fn parse_all_files_cancellable(
168    files: &[DiscoveredFile],
169    cache: Option<&CacheStore>,
170    need_complexity: bool,
171    cancellation: Option<&AtomicBool>,
172) -> ParseResult {
173    let parse_one = |file: &DiscoveredFile| {
174        if cancellation.is_some_and(|cancelled| cancelled.load(Ordering::SeqCst)) {
175            return ParseFileResult::default();
176        }
177        parse_single_file_cached(file, cache, need_complexity)
178    };
179    let results: Vec<ParseFileResult> = if files.len() <= PARALLEL_PARSE_FILE_THRESHOLD {
180        files.iter().map(parse_one).collect()
181    } else {
182        files.par_iter().map(parse_one).collect()
183    };
184
185    let mut modules = Vec::with_capacity(results.len());
186    let mut read_failures = Vec::new();
187    let mut parse_degradations = Vec::new();
188    let mut hits = 0usize;
189    let mut misses = 0usize;
190    let mut parse_cpu_nanos = 0u64;
191
192    // `results` is a positional map over `files`, so zipping recovers the path
193    // for a module without carrying one on `ModuleInfo`.
194    for (file, result) in files.iter().zip(results) {
195        hits += result.cache_hits;
196        misses += result.cache_misses;
197        parse_cpu_nanos = parse_cpu_nanos.saturating_add(result.parse_cpu_nanos);
198        if let Some(module) = result.module {
199            if module.parse_error_count > 0 {
200                parse_degradations.push(SourceParseDegradation {
201                    file_id: module.file_id,
202                    path: file.path.clone(),
203                    error_count: module.parse_error_count,
204                    panicked: module.parse_panicked,
205                });
206            }
207            modules.push(module);
208        }
209        if let Some(failure) = result.read_failure {
210            read_failures.push(failure);
211        }
212    }
213
214    if hits > 0 || misses > 0 {
215        tracing::info!(
216            cache_hits = hits,
217            cache_misses = misses,
218            "incremental cache stats"
219        );
220    }
221
222    ParseResult {
223        modules,
224        read_failures,
225        parse_degradations,
226        cache_hits: hits,
227        cache_misses: misses,
228        parse_cpu_ms: parse_cpu_nanos as f64 / 1_000_000.0,
229    }
230}
231
232#[derive(Default)]
233struct ParseFileResult {
234    module: Option<ModuleInfo>,
235    read_failure: Option<SourceReadFailure>,
236    cache_hits: usize,
237    cache_misses: usize,
238    parse_cpu_nanos: u64,
239}
240
241impl ParseFileResult {
242    fn cache_hit(module: ModuleInfo) -> Self {
243        Self {
244            module: Some(module),
245            read_failure: None,
246            cache_hits: 1,
247            cache_misses: 0,
248            parse_cpu_nanos: 0,
249        }
250    }
251
252    fn cache_miss(module: ModuleInfo, parse_cpu_nanos: u64) -> Self {
253        Self {
254            module: Some(module),
255            read_failure: None,
256            cache_hits: 0,
257            cache_misses: 1,
258            parse_cpu_nanos,
259        }
260    }
261
262    fn read_failure(file: &DiscoveredFile, error: &std::io::Error) -> Self {
263        Self {
264            module: None,
265            read_failure: Some(SourceReadFailure {
266                file_id: file.id,
267                path: file.path.clone(),
268                error: error.to_string(),
269            }),
270            cache_hits: 0,
271            cache_misses: 0,
272            parse_cpu_nanos: 0,
273        }
274    }
275}
276
277/// Parse a single file, consulting the cache first.
278///
279/// Cache validation strategy (fast path -> slow path):
280/// 1. Open the file so unreadable sources cannot use stale cached analysis
281/// 2. Read mtime + ctime + size from the open handle
282/// 3. If all three match the cached entry -> cache hit, return immediately
283/// 4. Otherwise -> read file, compute content hash
284/// 5. If content hash matches cached entry -> cache hit (file was rewritten or
285///    `touch`ed but its content is unchanged)
286/// 6. Otherwise -> cache miss, full parse
287///
288/// Step 3 requires ctime as well as mtime because mtime is writer-controlled:
289/// a same-length rewrite whose mtime is restored (`touch -r`, a codemod, a
290/// `git checkout` of an equal-length revision) leaves `(mtime, size)`
291/// unchanged, and serving the cached module for it means reporting the OLD
292/// file's unused exports with an auto-fixable `remove-export` action. A file
293/// whose ctime moved falls through to step 4 and still hits on the content
294/// hash, so the cost of the stricter gate is one read, not a reparse.
295fn parse_single_file_cached(
296    file: &DiscoveredFile,
297    cache: Option<&CacheStore>,
298    need_complexity: bool,
299) -> ParseFileResult {
300    let cached_by_path = cache.and_then(|store| store.get_by_path_only(&file.path));
301
302    if let Some(cached) = cached_by_path
303        && cached.file_size == file.size_bytes
304    {
305        let source_file = match std::fs::File::open(&file.path) {
306            Ok(source_file) => source_file,
307            Err(error) => return ParseFileResult::read_failure(file, &error),
308        };
309        if let Ok(metadata) = source_file.metadata()
310            && metadata.len() == cached.file_size
311        {
312            let fingerprint =
313                fallow_types::source_fingerprint::SourceFingerprint::from_metadata(&metadata);
314            if cached.source_fingerprint() == fingerprint
315                && fingerprint.is_trustworthy_without_content()
316                && (!need_complexity || cached.complexity_extracted)
317            {
318                return ParseFileResult::cache_hit(cache::cached_to_module_opts(
319                    cached,
320                    file.id,
321                    need_complexity,
322                ));
323            }
324        }
325    }
326
327    let raw = match std::fs::read_to_string(&file.path) {
328        Ok(raw) => raw,
329        Err(error) => return ParseFileResult::read_failure(file, &error),
330    };
331    let source = strip_bom(&raw);
332    let content_hash = xxhash_rust::xxh3::xxh3_64(source.as_bytes());
333
334    if let Some(cached) = cached_by_path
335        && cached.content_hash == content_hash
336        && (!need_complexity || cached.complexity_extracted)
337    {
338        return ParseFileResult::cache_hit(cache::cached_to_module_opts(
339            cached,
340            file.id,
341            need_complexity,
342        ));
343    }
344
345    let parse_start = std::time::Instant::now();
346    let module = parse_source_to_module(file.id, &file.path, source, content_hash, need_complexity);
347    let parse_cpu_nanos = u64::try_from(parse_start.elapsed().as_nanos()).unwrap_or(u64::MAX);
348    ParseFileResult::cache_miss(module, parse_cpu_nanos)
349}
350
351/// Parse a single file and extract module information (without complexity).
352#[must_use]
353pub fn parse_single_file(file: &DiscoveredFile) -> Option<ModuleInfo> {
354    let raw = std::fs::read_to_string(&file.path).ok()?;
355    let source = strip_bom(&raw);
356    let content_hash = xxhash_rust::xxh3::xxh3_64(source.as_bytes());
357    Some(parse_source_to_module(
358        file.id,
359        &file.path,
360        source,
361        content_hash,
362        false,
363    ))
364}
365
366/// Parse from in-memory content (for LSP, includes complexity).
367#[must_use]
368pub fn parse_from_content(file_id: FileId, path: &Path, content: &str) -> ModuleInfo {
369    let content = strip_bom(content);
370    let content_hash = xxhash_rust::xxh3::xxh3_64(content.as_bytes());
371    parse_source_to_module(file_id, path, content, content_hash, true)
372}
373
374#[cfg(all(test, not(miri)))]
375mod tests;
fallow_extract/lib.rs

fallow_extract/
lib.rs
