fallow-extract 3.46.0

AST extraction engine for fallow codebase intelligence (parser, complexity, SFC / Astro / MDX / CSS)
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
//! Parsing and extraction engine for fallow codebase intelligence.
//!
//! This crate handles all file parsing: JS/TS via Oxc, Vue/Svelte SFC extraction,
//! Astro frontmatter, MDX import/export extraction, CSS Module class name extraction,
//! HTML asset reference extraction, and incremental caching of parse results.
//!
//! User documentation for the supported file types is on [fallow.tools](https://fallow.tools/docs/analysis/file-types/):
//!
//! - [Supported file types](https://fallow.tools/docs/analysis/file-types/)
//! - [CSS analysis](https://fallow.tools/docs/analysis/css-analysis/)
//! - [Framework plugins](https://fallow.tools/docs/frameworks/built-in/)

#![warn(missing_docs)]
#![cfg_attr(not(test), deny(clippy::disallowed_methods))]
#![cfg_attr(
    test,
    allow(
        clippy::unwrap_used,
        clippy::expect_used,
        reason = "tests use unwrap and expect to keep fixture setup concise"
    )
)]

pub mod alchemy;
mod asset_url;
pub mod astro;
mod binding_references;
pub mod cache;
pub(crate) mod complexity;
mod complexity_names;
mod component_contracts;
pub(crate) mod cost;
pub mod css;
pub mod css_classes;
mod css_composes;
pub mod css_in_js;
mod css_mask;
mod css_masked_bytes;
pub mod css_metrics;
mod css_module_classes;
mod css_url;
pub mod dom_classes;
pub mod effect_facts;
mod effect_schema;
pub mod federation_runtime;
pub mod flags;
mod function_body;
mod function_contracts;
mod function_hash;
mod generated_file;
pub mod glimmer;
pub(crate) mod graphql;
pub(crate) mod html;
mod iconify;
pub mod inventory;
pub mod js_text;
mod jsdoc_attach;
mod jsdoc_deprecated;
mod literal_membership;
pub mod mdx;
mod module_info;
mod module_parse;
mod namespace_member_uses;
mod nesting;
pub mod og_image;
mod opaque_loads;
mod parse;
mod parse_degradation;
mod parse_options;
pub mod sfc;
pub mod sfc_css;
mod sfc_props;
mod sfc_template;
mod shared_state;
pub mod similar_code;
mod sink_identity;
mod source_map;
mod stack;
mod style_tokens;
pub mod suppress;
mod suppress_literals;
/// Tailwind CSS arbitrary-value detection.
pub mod tailwind;
pub(crate) mod template_complexity;
mod template_expression_scan;
mod template_usage;
/// Visitor utilities for AST extraction.
pub mod visitor;

use std::path::Path;
use std::sync::atomic::{AtomicBool, Ordering};

use rayon::prelude::*;

use cache::CacheStore;
use fallow_types::discover::{DiscoveredFile, FileId};

pub use fallow_types::extract::{
    AngularComponentFieldArrayTypeFact, AngularTemplateMemberAccessFact, AngularThisSpreadFact,
    ClassHeritageInfo, ClassThisMemberAccessFact, ClassThisWholeObjectUseFact,
    ComputedEnumKeyUseFact, DefaultImportWholeObjectUseFact, DynamicCustomElementRenderFact,
    DynamicImportInfo, DynamicImportPattern, ExportInfo, ExportName,
    ExportedObjectInstancePropertyFact, FactoryCallMemberAccessFact, FactoryFnMemberAccessFact,
    FactoryFnWholeObjectFact, FactoryPropertySource, FactoryReturnExport,
    FactoryReturnObjectPropertyAccessFact, FactoryReturnObjectPropertySourceFact,
    FactoryReturnObjectShapeExport, FlagPatterns, FluentChainMemberAccessFact,
    FluentChainNewMemberAccessFact, ImportInfo, ImportedName, InstanceExportBindingFact,
    LocalTypeDeclaration, MemberAccess, MemberInfo, MemberKind, ModuleInfo, ModuleLoadMechanism,
    ParseResult, PlaywrightConditionalFixtureDefinitionFact, PlaywrightFixtureAliasFact,
    PlaywrightFixtureDefinitionFact, PlaywrightFixtureTypeFact, PlaywrightFixtureUseFact,
    PlaywrightTestBindingFact, PublicSignatureTypeReference, QualifiedClassMemberAccessFact,
    ReExportInfo, RequireCallInfo, RequiredTypeMemberFact, SemanticFact, SourceParseDegradation,
    SourceReadFailure, StringEnumMemberValueFact, TypeAliasSurfaceTargetFact, TypeMemberTypeEntry,
    TypedPropertyMemberAccessFact, VisibilityTag, VitestModuleMockAction,
    VitestModuleMockOperationFact, compute_line_offsets,
};

pub use astro::{
    extract_astro_frontmatter, extract_astro_style_regions, extract_astro_template_regions,
};
pub use css::{
    StylesheetTokens, ThemeScan, ThemeTokenDef, extract_apply_tokens, extract_apply_tokens_located,
    extract_css_module_exports, extract_css_var_reads_located, scan_stylesheet_tokens,
    scan_theme_blocks,
};
pub use css_classes::{
    MarkupClassScan, MarkupClassToken, is_edit_distance_one, is_typo_edit, scan_markup_class_tokens,
};
pub use css_in_js::{
    ConsumerQuery, CssInJsObjectSheets, CssInJsToken, CssInJsTokenDef, CssInJsTokenOrigin,
    TokenConsumerHit, css_in_js_consumer_scan, css_in_js_object_sheets, css_in_js_theme_token_defs,
    css_in_js_token_defs, css_in_js_virtual_stylesheet,
};
pub use css_metrics::{compute_css_analytics, parse_css_color_rgb};
pub use dom_classes::{DomClassScan, scan_dom_class_values};
pub use function_contracts::{ParameterCondition, parameter_conditions};
pub use generated_file::is_generated_file;
pub use glimmer::{is_glimmer_file, strip_glimmer_templates};
pub use js_text::JsText;
pub use mdx::{extract_mdx_statements, extract_mdx_statements_mapped};
pub(crate) use nesting::parse_style_bounded;
pub use nesting::{MAX_NESTING_DEPTH, parse_bounded, parse_bounded_or_default};
pub use sfc::{
    SourceRegion, extract_sfc_scripts, extract_sfc_styles, extract_sfc_template_regions,
    is_sfc_file, sfc_merged_script_source,
};
pub use sfc_css::{
    scoped_unused_classes, sfc_preprocessor_virtual_stylesheet, sfc_virtual_stylesheet,
};
pub use similar_code::extract_similar_code_functions;
pub use sink_identity::sink_identity_text;
pub use source_map::{ExtractionResult, ScriptSection, embedded_script_sections};
pub use stack::register_thread_stack;
pub use suppress_literals::parse_suppressions_for_file;
pub use tailwind::{TailwindArbitraryUse, scan_tailwind_arbitrary_values};
pub use template_usage::TEMPLATE_SINK_CALLEE_PREFIX;

#[expect(
    clippy::expect_used,
    reason = "static regex patterns are hard-coded analyzer invariants covered by extraction tests"
)]
fn static_regex(pattern: &str) -> regex::Regex {
    regex::Regex::new(pattern).expect("static regex pattern should compile")
}

#[cfg(test)]
use module_parse::parse_source_to_module_with_mode;
pub use module_parse::{parse_source_to_module, parse_source_to_module_with_flags};

/// Leading UTF-8 byte order mark codepoint.
///
/// Windows editors (Notepad, older VS settings, some IDE plugins) emit a UTF-8
/// BOM at the start of source files. fallow's contract is "UTF-8 with or
/// without BOM; line offsets are computed against the post-BOM view; the BOM,
/// if present on input, is preserved on output by `fallow fix`."
const BOM_CHAR: char = '\u{FEFF}';
// Small, cache-hot inputs are faster on one thread than through Rayon setup.
// Larger file sets still use parallel parsing where parse work dominates.
const PARALLEL_PARSE_FILE_THRESHOLD: usize = 32;

/// Strip the leading UTF-8 BOM if present.
///
/// Called at every file-read entry point in this crate so the rest of the
/// pipeline (content hash, `compute_line_offsets`, oxc parser, downstream
/// analyses) sees a consistent post-BOM view. Mirrors the
/// `fallow_config` layer (`config_writer.rs::BOM`) so config-shaped sources
/// and source-code-shaped sources are processed symmetrically. See issue #475.
///
/// Call it once for each source. Only the first U+FEFF of a file is the
/// mark, so a second call removes a U+FEFF that is source text.
#[must_use]
fn strip_bom(source: &str) -> &str {
    source.strip_prefix(BOM_CHAR).unwrap_or(source)
}

/// The optional facts that a parse run must give for each module.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct ParseNeeds {
    /// Per-function complexity, for the `health` command.
    pub complexity: bool,
    /// The call facts (`ModuleInfo::function_contracts`), for the rules that
    /// read them.
    pub call_facts: bool,
}

/// Parse all files, extracting imports and exports.
///
/// Small file sets use a sequential fast path to avoid parallel scheduling
/// overhead; larger file sets use parallel extraction.
/// Uses the cache to skip reparsing files whose content hasn't changed.
///
/// When `need_complexity` is true, per-function cyclomatic/cognitive complexity
/// metrics are computed during parsing (needed by the `health` command).
/// Pass `false` for dead-code analysis where complexity data is unused.
///
/// Flag detection uses the built-in patterns only. A caller with a resolved
/// config uses [`parse_all_files_cancellable`] with the config's patterns,
/// because the cache keys on them.
///
/// The modules carry no call facts, as in a run of the default config. A
/// caller that reads them uses [`parse_all_files_with_mode`].
pub fn parse_all_files(
    files: &[DiscoveredFile],
    cache: Option<&CacheStore>,
    need_complexity: bool,
) -> ParseResult {
    parse_all_files_cancellable(
        files,
        cache,
        need_complexity,
        None,
        &FlagPatterns::default(),
    )
}

/// Parse all files, abandoning the remaining ones once `cancellation` is set.
///
/// Rayon's `map`/`collect` cannot short-circuit, so cancellation makes the
/// per-file body a no-op instead of stopping the iteration: the scheduled
/// items still drain, but at one atomic load each. The returned
/// [`ParseResult`] is therefore truncated whenever the token flipped, and
/// callers must treat a set token as a failed run rather than as a project
/// with fewer modules.
///
/// `flag_patterns` are the user flag patterns that detection applies on top
/// of the built-in ones. They must match the patterns the cache was keyed on.
/// The modules carry no call facts, as for [`parse_all_files`].
pub fn parse_all_files_cancellable(
    files: &[DiscoveredFile],
    cache: Option<&CacheStore>,
    need_complexity: bool,
    cancellation: Option<&AtomicBool>,
    flag_patterns: &FlagPatterns,
) -> ParseResult {
    parse_all_files_with_mode(
        files,
        cache,
        ParseNeeds {
            complexity: need_complexity,
            call_facts: false,
        },
        cancellation,
        flag_patterns,
        fallow_types::shared_state::ExtractionMode::Standard,
    )
}

/// Parse files with an explicit optional-fact mode matching the cache identity.
///
/// When `needs.call_facts` is `false`, the returned modules carry no call
/// facts (`function_contracts` is `None`), and a cache hit does not decode
/// them. When it is `true`, a cache entry that did not record call facts is a
/// miss, so each returned module has its call facts.
pub fn parse_all_files_with_mode(
    files: &[DiscoveredFile],
    cache: Option<&CacheStore>,
    needs: ParseNeeds,
    cancellation: Option<&AtomicBool>,
    flag_patterns: &FlagPatterns,
    mode: fallow_types::shared_state::ExtractionMode,
) -> ParseResult {
    let parse_one = |file: &DiscoveredFile| {
        if cancellation.is_some_and(|cancelled| cancelled.load(Ordering::SeqCst)) {
            return ParseFileResult::default();
        }
        parse_single_file_cached(file, cache, needs, flag_patterns, mode)
    };
    let results: Vec<ParseFileResult> = if files.len() <= PARALLEL_PARSE_FILE_THRESHOLD {
        files.iter().map(parse_one).collect()
    } else {
        files.par_iter().map(parse_one).collect()
    };

    let mut modules = Vec::with_capacity(results.len());
    let mut read_failures = Vec::new();
    let mut parse_degradations = Vec::new();
    let mut hits = 0usize;
    let mut misses = 0usize;
    let mut parse_cpu_nanos = 0u64;
    let mut files_read = 0u64;
    let mut source_bytes_read = 0u64;
    let mut css_masked_bytes = 0u64;

    // `results` is a positional map over `files`, so zipping recovers the path
    // for a module without carrying one on `ModuleInfo`.
    for (file, result) in files.iter().zip(results) {
        hits += result.cache_hits;
        misses += result.cache_misses;
        parse_cpu_nanos = parse_cpu_nanos.saturating_add(result.parse_cpu_nanos);
        if let Some(bytes) = result.source_bytes_read {
            files_read += 1;
            source_bytes_read += bytes;
        }
        css_masked_bytes += result.css_masked_bytes;
        if let Some(module) = result.module {
            if module.parse_error_count > 0 {
                parse_degradations.push(SourceParseDegradation {
                    file_id: module.file_id,
                    path: file.path.clone(),
                    error_count: module.parse_error_count,
                    panicked: module.parse_panicked,
                    nesting_limit: module.parse_nesting_too_deep.then_some(MAX_NESTING_DEPTH),
                });
            }
            modules.push(module);
        }
        if let Some(failure) = result.read_failure {
            read_failures.push(failure);
        }
    }

    if hits > 0 || misses > 0 {
        tracing::info!(
            cache_hits = hits,
            cache_misses = misses,
            "incremental cache stats"
        );
    }

    ParseResult {
        modules,
        read_failures,
        parse_degradations,
        cache_hits: hits,
        cache_misses: misses,
        parse_cpu_ms: parse_cpu_nanos as f64 / 1_000_000.0,
        files_read,
        source_bytes_read,
        css_masked_bytes,
    }
}

#[derive(Default)]
struct ParseFileResult {
    module: Option<ModuleInfo>,
    read_failure: Option<SourceReadFailure>,
    cache_hits: usize,
    cache_misses: usize,
    parse_cpu_nanos: u64,
    /// Source bytes read from disk for this file, or `None` when the file was
    /// served from cache metadata without a read.
    source_bytes_read: Option<u64>,
    /// Source bytes that the CSS comment mask read during the parse.
    css_masked_bytes: u64,
}

impl ParseFileResult {
    fn cache_hit(module: ModuleInfo) -> Self {
        Self {
            module: Some(module),
            read_failure: None,
            cache_hits: 1,
            cache_misses: 0,
            parse_cpu_nanos: 0,
            source_bytes_read: None,
            css_masked_bytes: 0,
        }
    }

    fn cache_miss(module: ModuleInfo, parse_cpu_nanos: u64) -> Self {
        Self {
            module: Some(module),
            read_failure: None,
            cache_hits: 0,
            cache_misses: 1,
            parse_cpu_nanos,
            source_bytes_read: None,
            css_masked_bytes: 0,
        }
    }

    const fn with_source_bytes_read(mut self, bytes: usize) -> Self {
        self.source_bytes_read = Some(bytes as u64);
        self
    }

    fn read_failure(file: &DiscoveredFile, error: &std::io::Error) -> Self {
        Self {
            module: None,
            read_failure: Some(SourceReadFailure {
                file_id: file.id,
                path: file.path.clone(),
                error: error.to_string(),
            }),
            cache_hits: 0,
            cache_misses: 0,
            parse_cpu_nanos: 0,
            source_bytes_read: None,
            css_masked_bytes: 0,
        }
    }
}

/// Parse a single file, consulting the cache first.
///
/// Cache validation strategy (fast path -> slow path):
/// 1. Open the file so unreadable sources cannot use stale cached analysis
/// 2. Read mtime + ctime + size from the open handle
/// 3. If all three match the cached entry -> cache hit, return immediately
/// 4. Otherwise -> read file, compute content hash
/// 5. If content hash matches cached entry -> cache hit (file was rewritten or
///    `touch`ed but its content is unchanged)
/// 6. Otherwise -> cache miss, full parse
///
/// Step 3 requires ctime as well as mtime because mtime is writer-controlled:
/// a same-length rewrite whose mtime is restored (`touch -r`, a codemod, a
/// `git checkout` of an equal-length revision) leaves `(mtime, size)`
/// unchanged, and serving the cached module for it means reporting the OLD
/// file's unused exports with an auto-fixable `remove-export` action. A file
/// whose ctime moved falls through to step 4 and still hits on the content
/// hash, so the cost of the stricter gate is one read, not a reparse.
///
/// Step 3 also needs a stored fingerprint that settled before the run that
/// wrote it read the file. A same-length write in the same timestamp tick as
/// the previous write keeps mtime, ctime and size, so the cache writer stores
/// a recent fingerprint without its ctime
/// (`SourceFingerprint::for_content_read_at`). Such an entry never matches at
/// step 3, and the next run compares the content hash.
fn parse_single_file_cached(
    file: &DiscoveredFile,
    cache: Option<&CacheStore>,
    needs: ParseNeeds,
    flag_patterns: &FlagPatterns,
    mode: fallow_types::shared_state::ExtractionMode,
) -> ParseFileResult {
    let ParseNeeds {
        complexity: need_complexity,
        call_facts: need_call_facts,
    } = needs;
    let cached_by_path = cache.and_then(|store| store.get_by_path_only(&file.path));

    if let Some(cached) = cached_by_path
        && cached.file_size == file.size_bytes
    {
        let source_file = match std::fs::File::open(&file.path) {
            Ok(source_file) => source_file,
            Err(error) => return ParseFileResult::read_failure(file, &error),
        };
        if let Ok(metadata) = source_file.metadata()
            && metadata.len() == cached.file_size
        {
            let fingerprint =
                fallow_types::source_fingerprint::SourceFingerprint::from_metadata(&metadata);
            if cached.source_fingerprint() == fingerprint
                && fingerprint.is_trustworthy_without_content()
                && (!need_complexity || cached.complexity_extracted)
                && let Some(module) =
                    cache::cached_hit_to_module(cached, file.id, need_complexity, need_call_facts)
            {
                return ParseFileResult::cache_hit(module);
            }
        }
    }

    let raw = match std::fs::read_to_string(&file.path) {
        Ok(raw) => raw,
        Err(error) => return ParseFileResult::read_failure(file, &error),
    };
    let source = strip_bom(&raw);
    let content_hash = xxhash_rust::xxh3::xxh3_64(source.as_bytes());

    if let Some(cached) = cached_by_path
        && cached.content_hash == content_hash
        && (!need_complexity || cached.complexity_extracted)
        && let Some(module) =
            cache::cached_hit_to_module(cached, file.id, need_complexity, need_call_facts)
    {
        return ParseFileResult::cache_hit(module).with_source_bytes_read(raw.len());
    }

    let parse_start = std::time::Instant::now();
    // Drop a count that a scan outside a parse left on this thread.
    css_masked_bytes::take();
    let mut module = module_parse::parse_source_without_bom(
        file.id,
        &file.path,
        source,
        content_hash,
        need_complexity,
        &module_parse::ExtractionOptions {
            flag_patterns,
            mode,
        },
    );
    if !need_call_facts {
        // A cache hit of the same run gives no call facts, so a parsed module
        // must not give them either.
        module.function_contracts = None;
    }
    let parse_cpu_nanos = u64::try_from(parse_start.elapsed().as_nanos()).unwrap_or(u64::MAX);
    let mut result =
        ParseFileResult::cache_miss(module, parse_cpu_nanos).with_source_bytes_read(raw.len());
    result.css_masked_bytes = css_masked_bytes::take();
    result
}

/// Parse a single file and extract module information (without complexity).
#[must_use]
pub fn parse_single_file(file: &DiscoveredFile) -> Option<ModuleInfo> {
    let raw = std::fs::read_to_string(&file.path).ok()?;
    let source = strip_bom(&raw);
    let content_hash = xxhash_rust::xxh3::xxh3_64(source.as_bytes());
    Some(module_parse::parse_source_without_bom(
        file.id,
        &file.path,
        source,
        content_hash,
        false,
        &module_parse::ExtractionOptions {
            flag_patterns: &FlagPatterns::default(),
            mode: fallow_types::shared_state::ExtractionMode::Standard,
        },
    ))
}

/// Parse from in-memory content (for LSP, includes complexity).
#[must_use]
pub fn parse_from_content(file_id: FileId, path: &Path, content: &str) -> ModuleInfo {
    let content = strip_bom(content);
    let content_hash = xxhash_rust::xxh3::xxh3_64(content.as_bytes());
    module_parse::parse_source_without_bom(
        file_id,
        path,
        content,
        content_hash,
        true,
        &module_parse::ExtractionOptions {
            flag_patterns: &FlagPatterns::default(),
            mode: fallow_types::shared_state::ExtractionMode::Standard,
        },
    )
}

#[cfg(all(test, not(miri)))]
mod tests;
#[cfg(all(test, not(miri)))]
mod unicode_span_oracle;