fallow_extract/
lib.rs

1//! Parsing and extraction engine for fallow codebase intelligence.
2//!
3//! This crate handles all file parsing: JS/TS via Oxc, Vue/Svelte SFC extraction,
4//! Astro frontmatter, MDX import/export extraction, CSS Module class name extraction,
5//! HTML asset reference extraction, and incremental caching of parse results.
6
7#![warn(missing_docs)]
8#![cfg_attr(not(test), deny(clippy::disallowed_methods))]
9#![cfg_attr(
10    test,
11    allow(
12        clippy::unwrap_used,
13        clippy::expect_used,
14        reason = "tests use unwrap and expect to keep fixture setup concise"
15    )
16)]
17
18mod asset_url;
19pub mod astro;
20pub mod cache;
21pub(crate) mod complexity;
22pub mod css;
23pub mod css_classes;
24pub mod css_in_js;
25pub mod css_metrics;
26pub mod flags;
27pub mod glimmer;
28pub(crate) mod graphql;
29pub(crate) mod html;
30pub(crate) mod iconify;
31pub mod inventory;
32pub mod mdx;
33mod module_info;
34mod parse;
35pub mod sfc;
36pub mod sfc_css;
37mod sfc_props;
38mod sfc_template;
39mod source_map;
40pub mod suppress;
41/// Tailwind CSS arbitrary-value detection.
42pub mod tailwind;
43pub(crate) mod template_complexity;
44mod template_expression_scan;
45mod template_usage;
46/// Visitor utilities for AST extraction.
47pub mod visitor;
48
49use std::path::Path;
50
51use rayon::prelude::*;
52
53use cache::CacheStore;
54use fallow_types::discover::{DiscoveredFile, FileId};
55
56pub use fallow_types::extract::{
57    AngularComponentFieldArrayTypeFact, AngularTemplateMemberAccessFact, AngularThisSpreadFact,
58    ClassHeritageInfo, ClassThisMemberAccessFact, ClassThisWholeObjectUseFact,
59    ComputedEnumKeyUseFact, DefaultImportWholeObjectUseFact, DynamicCustomElementRenderFact,
60    DynamicImportInfo, DynamicImportPattern, ExportInfo, ExportName, FactoryCallMemberAccessFact,
61    FactoryFnMemberAccessFact, FactoryFnWholeObjectFact, FactoryReturnExport,
62    FactoryReturnObjectPropertyAccessFact, FactoryReturnObjectShapeExport,
63    FluentChainMemberAccessFact, FluentChainNewMemberAccessFact, ImportInfo, ImportedName,
64    InstanceExportBindingFact, LocalTypeDeclaration, MemberAccess, MemberInfo, MemberKind,
65    ModuleInfo, ModuleLoadMechanism, ParseResult, PlaywrightFixtureAliasFact,
66    PlaywrightFixtureDefinitionFact, PlaywrightFixtureTypeFact, PlaywrightFixtureUseFact,
67    PublicSignatureTypeReference, ReExportInfo, RequireCallInfo, RequiredTypeMemberFact,
68    SemanticFact, SourceReadFailure, StringEnumMemberValueFact, TypeAliasSurfaceTargetFact,
69    TypeMemberTypeEntry, TypedPropertyMemberAccessFact, VisibilityTag, VitestModuleMockAction,
70    VitestModuleMockOperationFact, compute_line_offsets,
71};
72
73pub use astro::{
74    extract_astro_frontmatter, extract_astro_style_regions, extract_astro_template_regions,
75};
76pub use css::{
77    ThemeScan, ThemeTokenDef, extract_apply_tokens, extract_apply_tokens_located,
78    extract_css_module_exports, extract_css_var_reads_located, scan_theme_blocks,
79};
80pub use css_classes::{
81    MarkupClassScan, MarkupClassToken, is_edit_distance_one, is_typo_edit, scan_markup_class_tokens,
82};
83pub use css_in_js::{
84    ConsumerQuery, CssInJsObjectSheets, CssInJsToken, CssInJsTokenDef, CssInJsTokenOrigin,
85    TokenConsumerHit, css_in_js_consumer_scan, css_in_js_object_sheets, css_in_js_theme_consumers,
86    css_in_js_theme_token_defs, css_in_js_token_consumers, css_in_js_token_defs,
87    css_in_js_virtual_stylesheet, panda_style_value_consumers, panda_token_call_consumers,
88};
89pub use css_metrics::{compute_css_analytics, parse_css_color_rgb};
90pub use glimmer::{is_glimmer_file, strip_glimmer_templates};
91pub use mdx::{extract_mdx_statements, extract_mdx_statements_mapped};
92pub use sfc::{
93    SourceRegion, extract_sfc_scripts, extract_sfc_styles, extract_sfc_template_regions,
94    is_sfc_file,
95};
96pub use sfc_css::{
97    scoped_unused_classes, sfc_preprocessor_virtual_stylesheet, sfc_virtual_stylesheet,
98};
99pub use source_map::ExtractionResult;
100pub use tailwind::{TailwindArbitraryUse, scan_tailwind_arbitrary_values};
101
102#[expect(
103    clippy::expect_used,
104    reason = "static regex patterns are hard-coded analyzer invariants covered by extraction tests"
105)]
106fn static_regex(pattern: &str) -> regex::Regex {
107    regex::Regex::new(pattern).expect("static regex pattern should compile")
108}
109
110pub use parse::parse_source_to_module;
111
112/// Leading UTF-8 byte order mark codepoint.
113///
114/// Windows editors (Notepad, older VS settings, some IDE plugins) emit a UTF-8
115/// BOM at the start of source files. fallow's contract is "UTF-8 with or
116/// without BOM; line offsets are computed against the post-BOM view; the BOM,
117/// if present on input, is preserved on output by `fallow fix`."
118const BOM_CHAR: char = '\u{FEFF}';
119// Small, cache-hot inputs are faster on one thread than through Rayon setup.
120// Larger file sets still use parallel parsing where parse work dominates.
121const PARALLEL_PARSE_FILE_THRESHOLD: usize = 32;
122
123/// Strip the leading UTF-8 BOM if present.
124///
125/// Called at every file-read entry point in this crate so the rest of the
126/// pipeline (content hash, `compute_line_offsets`, oxc parser, downstream
127/// analyses) sees a consistent post-BOM view. Mirrors the
128/// `fallow_config` layer (`config_writer.rs::BOM`) so config-shaped sources
129/// and source-code-shaped sources are processed symmetrically. See issue #475.
130#[must_use]
131fn strip_bom(source: &str) -> &str {
132    source.strip_prefix(BOM_CHAR).unwrap_or(source)
133}
134
135/// Parse all files, extracting imports and exports.
136///
137/// Small file sets use a sequential fast path to avoid parallel scheduling
138/// overhead; larger file sets use parallel extraction.
139/// Uses the cache to skip reparsing files whose content hasn't changed.
140///
141/// When `need_complexity` is true, per-function cyclomatic/cognitive complexity
142/// metrics are computed during parsing (needed by the `health` command).
143/// Pass `false` for dead-code analysis where complexity data is unused.
144pub fn parse_all_files(
145    files: &[DiscoveredFile],
146    cache: Option<&CacheStore>,
147    need_complexity: bool,
148) -> ParseResult {
149    let results: Vec<ParseFileResult> = if files.len() <= PARALLEL_PARSE_FILE_THRESHOLD {
150        files
151            .iter()
152            .map(|file| parse_single_file_cached(file, cache, need_complexity))
153            .collect()
154    } else {
155        files
156            .par_iter()
157            .map(|file| parse_single_file_cached(file, cache, need_complexity))
158            .collect()
159    };
160
161    let mut modules = Vec::with_capacity(results.len());
162    let mut read_failures = Vec::new();
163    let mut hits = 0usize;
164    let mut misses = 0usize;
165    let mut parse_cpu_nanos = 0u64;
166
167    for result in results {
168        hits += result.cache_hits;
169        misses += result.cache_misses;
170        parse_cpu_nanos = parse_cpu_nanos.saturating_add(result.parse_cpu_nanos);
171        if let Some(module) = result.module {
172            modules.push(module);
173        }
174        if let Some(failure) = result.read_failure {
175            read_failures.push(failure);
176        }
177    }
178
179    if hits > 0 || misses > 0 {
180        tracing::info!(
181            cache_hits = hits,
182            cache_misses = misses,
183            "incremental cache stats"
184        );
185    }
186
187    ParseResult {
188        modules,
189        read_failures,
190        cache_hits: hits,
191        cache_misses: misses,
192        parse_cpu_ms: parse_cpu_nanos as f64 / 1_000_000.0,
193    }
194}
195
196struct ParseFileResult {
197    module: Option<ModuleInfo>,
198    read_failure: Option<SourceReadFailure>,
199    cache_hits: usize,
200    cache_misses: usize,
201    parse_cpu_nanos: u64,
202}
203
204impl ParseFileResult {
205    fn cache_hit(module: ModuleInfo) -> Self {
206        Self {
207            module: Some(module),
208            read_failure: None,
209            cache_hits: 1,
210            cache_misses: 0,
211            parse_cpu_nanos: 0,
212        }
213    }
214
215    fn cache_miss(module: ModuleInfo, parse_cpu_nanos: u64) -> Self {
216        Self {
217            module: Some(module),
218            read_failure: None,
219            cache_hits: 0,
220            cache_misses: 1,
221            parse_cpu_nanos,
222        }
223    }
224
225    fn read_failure(file: &DiscoveredFile, error: &std::io::Error) -> Self {
226        Self {
227            module: None,
228            read_failure: Some(SourceReadFailure {
229                file_id: file.id,
230                path: file.path.clone(),
231                error: error.to_string(),
232            }),
233            cache_hits: 0,
234            cache_misses: 0,
235            parse_cpu_nanos: 0,
236        }
237    }
238}
239
240/// Parse a single file, consulting the cache first.
241///
242/// Cache validation strategy (fast path -> slow path):
243/// 1. Open the file so unreadable sources cannot use stale cached analysis
244/// 2. Read mtime + size from the open handle
245/// 3. If mtime+size match the cached entry -> cache hit, return immediately
246/// 4. If mtime+size differ -> read file, compute content hash
247/// 5. If content hash matches cached entry -> cache hit (file was `touch`ed but unchanged)
248/// 6. Otherwise -> cache miss, full parse
249fn parse_single_file_cached(
250    file: &DiscoveredFile,
251    cache: Option<&CacheStore>,
252    need_complexity: bool,
253) -> ParseFileResult {
254    let cached_by_path = cache.and_then(|store| store.get_by_path_only(&file.path));
255
256    if let Some(cached) = cached_by_path
257        && cached.file_size == file.size_bytes
258    {
259        let source_file = match std::fs::File::open(&file.path) {
260            Ok(source_file) => source_file,
261            Err(error) => return ParseFileResult::read_failure(file, &error),
262        };
263        if let Ok(metadata) = source_file.metadata()
264            && metadata.len() == cached.file_size
265        {
266            let fingerprint =
267                fallow_types::source_fingerprint::SourceFingerprint::from_metadata(&metadata);
268            if cached.source_fingerprint() == fingerprint
269                && fingerprint.has_known_mtime()
270                && (!need_complexity || !cached.complexity.is_empty())
271            {
272                return ParseFileResult::cache_hit(cache::cached_to_module_opts(
273                    cached,
274                    file.id,
275                    need_complexity,
276                ));
277            }
278        }
279    }
280
281    let raw = match std::fs::read_to_string(&file.path) {
282        Ok(raw) => raw,
283        Err(error) => return ParseFileResult::read_failure(file, &error),
284    };
285    let source = strip_bom(&raw);
286    let content_hash = xxhash_rust::xxh3::xxh3_64(source.as_bytes());
287
288    if let Some(cached) = cached_by_path
289        && cached.content_hash == content_hash
290        && (!need_complexity || !cached.complexity.is_empty())
291    {
292        return ParseFileResult::cache_hit(cache::cached_to_module_opts(
293            cached,
294            file.id,
295            need_complexity,
296        ));
297    }
298
299    let parse_start = std::time::Instant::now();
300    let module = parse_source_to_module(file.id, &file.path, source, content_hash, need_complexity);
301    let parse_cpu_nanos = u64::try_from(parse_start.elapsed().as_nanos()).unwrap_or(u64::MAX);
302    ParseFileResult::cache_miss(module, parse_cpu_nanos)
303}
304
305/// Parse a single file and extract module information (without complexity).
306#[must_use]
307pub fn parse_single_file(file: &DiscoveredFile) -> Option<ModuleInfo> {
308    let raw = std::fs::read_to_string(&file.path).ok()?;
309    let source = strip_bom(&raw);
310    let content_hash = xxhash_rust::xxh3::xxh3_64(source.as_bytes());
311    Some(parse_source_to_module(
312        file.id,
313        &file.path,
314        source,
315        content_hash,
316        false,
317    ))
318}
319
320/// Parse from in-memory content (for LSP, includes complexity).
321#[must_use]
322pub fn parse_from_content(file_id: FileId, path: &Path, content: &str) -> ModuleInfo {
323    let content = strip_bom(content);
324    let content_hash = xxhash_rust::xxh3::xxh3_64(content.as_bytes());
325    parse_source_to_module(file_id, path, content, content_hash, true)
326}
327
328#[cfg(all(test, not(miri)))]
329mod tests;
fallow_extract/lib.rs

fallow_extract/
lib.rs
