fallow_extract/
similar_code.rs

1//! On-demand, bounded JS and TS function extraction for similar-code providers.
2
3use std::collections::BTreeSet;
4use std::path::Path;
5
6use fallow_types::similar_code::{
7    ExtractedSimilarCodeFunction, SIMILAR_CODE_EXTRACTION_SEMANTICS_VERSION, SimilarCodeExtraction,
8    SimilarCodeExtractionLimits, SimilarCodeExtractionSkip, SimilarCodeExtractionSkipReason,
9    SimilarCodeFunctionKind, SimilarCodeFunctionLocation, SimilarCodeSideEffectHint,
10    SimilarCodeSourceDigest,
11};
12use oxc_allocator::Allocator;
13use oxc_ast::ast::{
14    ArrowFunctionExpression, AssignmentExpression, AwaitExpression, CallExpression, Class,
15    ClassElement, ComputedMemberExpression, Declaration, ExportDefaultDeclarationKind, Function,
16    FunctionBody, ImportExpression, JSXElement, JSXFragment, MethodDefinitionKind, NewExpression,
17    ObjectExpression, ObjectPropertyKind, PrivateFieldExpression, Program, PropertyKind, Statement,
18    StaticMemberExpression, TaggedTemplateExpression, ThrowStatement, UnaryExpression,
19    UpdateExpression, VariableDeclaration, YieldExpression,
20};
21use oxc_ast_visit::{Visit, walk};
22use oxc_parser::Parser;
23use oxc_semantic::ScopeFlags;
24use oxc_span::{GetSpan, SourceType, Span};
25use rustc_hash::FxHashMap;
26use sha2::{Digest, Sha256};
27
28#[derive(Debug, Clone, Copy)]
29struct ReviewMetadata {
30    param_count: u32,
31    is_async: bool,
32    is_generator: bool,
33    has_await: bool,
34    has_throw: bool,
35    side_effect_hint: SimilarCodeSideEffectHint,
36}
37
38impl ReviewMetadata {
39    fn for_function(function: &Function<'_>, syntax_reliable: bool) -> Self {
40        Self::from_body(
41            function.body.as_deref(),
42            function.params.items.len(),
43            function.params.rest.is_some(),
44            function.r#async,
45            function.generator,
46            syntax_reliable,
47        )
48    }
49
50    fn for_arrow(arrow: &ArrowFunctionExpression<'_>, syntax_reliable: bool) -> Self {
51        Self::from_body(
52            Some(arrow.body.as_ref()),
53            arrow.params.items.len(),
54            arrow.params.rest.is_some(),
55            arrow.r#async,
56            false,
57            syntax_reliable,
58        )
59    }
60
61    fn from_body(
62        body: Option<&FunctionBody<'_>>,
63        fixed_params: usize,
64        has_rest: bool,
65        is_async: bool,
66        is_generator: bool,
67        syntax_reliable: bool,
68    ) -> Self {
69        let mut visitor = ReviewMetadataVisitor::default();
70        if let Some(body) = body {
71            visitor.visit_function_body(body);
72        }
73        let param_count = fixed_params.saturating_add(usize::from(has_rest));
74        let side_effect_hint = if !syntax_reliable {
75            SimilarCodeSideEffectHint::Unknown
76        } else if visitor.may_have_side_effects {
77            SimilarCodeSideEffectHint::MayHaveSideEffects
78        } else {
79            SimilarCodeSideEffectHint::PureLooking
80        };
81        Self {
82            param_count: u32::try_from(param_count).unwrap_or(u32::MAX),
83            is_async,
84            is_generator,
85            has_await: visitor.has_await,
86            has_throw: visitor.has_throw,
87            side_effect_hint,
88        }
89    }
90}
91
92#[derive(Default)]
93struct ReviewMetadataVisitor {
94    has_await: bool,
95    has_throw: bool,
96    may_have_side_effects: bool,
97}
98
99impl<'ast> Visit<'ast> for ReviewMetadataVisitor {
100    fn visit_function(&mut self, _function: &Function<'ast>, _flags: ScopeFlags) {}
101
102    fn visit_arrow_function_expression(&mut self, _arrow: &ArrowFunctionExpression<'ast>) {}
103
104    fn visit_call_expression(&mut self, expression: &CallExpression<'ast>) {
105        self.may_have_side_effects = true;
106        walk::walk_call_expression(self, expression);
107    }
108
109    fn visit_new_expression(&mut self, expression: &NewExpression<'ast>) {
110        self.may_have_side_effects = true;
111        walk::walk_new_expression(self, expression);
112    }
113
114    fn visit_assignment_expression(&mut self, expression: &AssignmentExpression<'ast>) {
115        self.may_have_side_effects = true;
116        walk::walk_assignment_expression(self, expression);
117    }
118
119    fn visit_update_expression(&mut self, expression: &UpdateExpression<'ast>) {
120        self.may_have_side_effects = true;
121        walk::walk_update_expression(self, expression);
122    }
123
124    fn visit_unary_expression(&mut self, expression: &UnaryExpression<'ast>) {
125        self.may_have_side_effects |= expression.operator.is_delete();
126        walk::walk_unary_expression(self, expression);
127    }
128
129    fn visit_await_expression(&mut self, expression: &AwaitExpression<'ast>) {
130        self.has_await = true;
131        self.may_have_side_effects = true;
132        walk::walk_await_expression(self, expression);
133    }
134
135    fn visit_throw_statement(&mut self, statement: &ThrowStatement<'ast>) {
136        self.has_throw = true;
137        self.may_have_side_effects = true;
138        walk::walk_throw_statement(self, statement);
139    }
140
141    fn visit_yield_expression(&mut self, expression: &YieldExpression<'ast>) {
142        self.may_have_side_effects = true;
143        walk::walk_yield_expression(self, expression);
144    }
145
146    fn visit_import_expression(&mut self, expression: &ImportExpression<'ast>) {
147        self.may_have_side_effects = true;
148        walk::walk_import_expression(self, expression);
149    }
150
151    fn visit_tagged_template_expression(&mut self, expression: &TaggedTemplateExpression<'ast>) {
152        self.may_have_side_effects = true;
153        walk::walk_tagged_template_expression(self, expression);
154    }
155
156    fn visit_computed_member_expression(&mut self, expression: &ComputedMemberExpression<'ast>) {
157        self.may_have_side_effects = true;
158        walk::walk_computed_member_expression(self, expression);
159    }
160
161    fn visit_static_member_expression(&mut self, expression: &StaticMemberExpression<'ast>) {
162        self.may_have_side_effects = true;
163        walk::walk_static_member_expression(self, expression);
164    }
165
166    fn visit_private_field_expression(&mut self, expression: &PrivateFieldExpression<'ast>) {
167        self.may_have_side_effects = true;
168        walk::walk_private_field_expression(self, expression);
169    }
170
171    fn visit_jsx_element(&mut self, element: &JSXElement<'ast>) {
172        self.may_have_side_effects = true;
173        walk::walk_jsx_element(self, element);
174    }
175
176    fn visit_jsx_fragment(&mut self, fragment: &JSXFragment<'ast>) {
177        self.may_have_side_effects = true;
178        walk::walk_jsx_fragment(self, fragment);
179    }
180}
181
182struct ExtractionBuilder<'a> {
183    file: String,
184    source: &'a str,
185    line_offsets: Vec<u32>,
186    limits: SimilarCodeExtractionLimits,
187    functions: Vec<ExtractedSimilarCodeFunction>,
188    source_bytes: usize,
189    syntax_reliable: bool,
190    classified_spans: BTreeSet<(u32, u32)>,
191    skips: FxHashMap<SimilarCodeExtractionSkipReason, usize>,
192}
193
194impl<'a> ExtractionBuilder<'a> {
195    fn new(
196        file: String,
197        source: &'a str,
198        limits: SimilarCodeExtractionLimits,
199        syntax_reliable: bool,
200    ) -> Self {
201        Self {
202            file,
203            source,
204            line_offsets: fallow_types::extract::compute_line_offsets(source),
205            limits,
206            functions: Vec::new(),
207            source_bytes: 0,
208            syntax_reliable,
209            classified_spans: BTreeSet::new(),
210            skips: FxHashMap::default(),
211        }
212    }
213
214    fn collect_program(&mut self, program: &Program<'_>) {
215        for statement in &program.body {
216            match statement {
217                Statement::FunctionDeclaration(function) => {
218                    self.collect_function_declaration(function, None);
219                }
220                Statement::VariableDeclaration(declaration) => {
221                    self.collect_variable_declaration(declaration);
222                }
223                Statement::ClassDeclaration(class) => {
224                    self.collect_class_declaration(class, None);
225                }
226                Statement::ExportNamedDeclaration(export) => {
227                    if let Some(declaration) = &export.declaration {
228                        self.collect_declaration(declaration);
229                    }
230                }
231                Statement::ExportDefaultDeclaration(export) => {
232                    self.collect_default_export(&export.declaration);
233                }
234                _ => {}
235            }
236        }
237    }
238
239    fn collect_declaration(&mut self, declaration: &Declaration<'_>) {
240        match declaration {
241            Declaration::FunctionDeclaration(function) => {
242                self.collect_function_declaration(function, None);
243            }
244            Declaration::VariableDeclaration(declaration) => {
245                self.collect_variable_declaration(declaration);
246            }
247            Declaration::ClassDeclaration(class) => {
248                self.collect_class_declaration(class, None);
249            }
250            _ => {}
251        }
252    }
253
254    fn collect_default_export(&mut self, declaration: &ExportDefaultDeclarationKind<'_>) {
255        match declaration {
256            ExportDefaultDeclarationKind::FunctionDeclaration(function) => {
257                self.collect_function_declaration(function, Some("default"));
258            }
259            ExportDefaultDeclarationKind::FunctionExpression(function) => {
260                self.collect_function_expression(function, "default");
261            }
262            ExportDefaultDeclarationKind::ArrowFunctionExpression(arrow) => {
263                self.collect_arrow_function(arrow, "default");
264            }
265            ExportDefaultDeclarationKind::ClassDeclaration(class)
266            | ExportDefaultDeclarationKind::ClassExpression(class) => {
267                self.collect_class_declaration(class, Some("default"));
268            }
269            ExportDefaultDeclarationKind::ObjectExpression(object) => {
270                self.collect_object_methods(object, "default");
271            }
272            _ => {}
273        }
274    }
275
276    fn collect_variable_declaration(&mut self, declaration: &VariableDeclaration<'_>) {
277        for declarator in &declaration.declarations {
278            let Some(initializer) = &declarator.init else {
279                continue;
280            };
281            let is_supported_container = matches!(
282                initializer,
283                oxc_ast::ast::Expression::FunctionExpression(_)
284                    | oxc_ast::ast::Expression::ArrowFunctionExpression(_)
285                    | oxc_ast::ast::Expression::ClassExpression(_)
286                    | oxc_ast::ast::Expression::ObjectExpression(_)
287            );
288            if !is_supported_container {
289                continue;
290            }
291
292            let Some(binding) = declarator.id.get_binding_identifier() else {
293                if matches!(
294                    initializer,
295                    oxc_ast::ast::Expression::FunctionExpression(_)
296                        | oxc_ast::ast::Expression::ArrowFunctionExpression(_)
297                ) {
298                    self.classified_spans
299                        .insert((initializer.span().start, initializer.span().end));
300                    self.record_skip(SimilarCodeExtractionSkipReason::UnsupportedFunctionForm, 1);
301                }
302                continue;
303            };
304            match initializer {
305                oxc_ast::ast::Expression::FunctionExpression(function) => {
306                    self.collect_function_expression(function, binding.name.as_str());
307                }
308                oxc_ast::ast::Expression::ArrowFunctionExpression(arrow) => {
309                    self.collect_arrow_function(arrow, binding.name.as_str());
310                }
311                oxc_ast::ast::Expression::ClassExpression(class) => {
312                    self.collect_class(class, binding.name.as_str());
313                }
314                oxc_ast::ast::Expression::ObjectExpression(object) => {
315                    self.collect_object_methods(object, binding.name.as_str());
316                }
317                _ => {}
318            }
319        }
320    }
321
322    fn collect_class_declaration(&mut self, class: &Class<'_>, fallback: Option<&str>) {
323        let Some(name) = class
324            .id
325            .as_ref()
326            .map(|identifier| identifier.name.as_str())
327            .or(fallback)
328        else {
329            return;
330        };
331        self.collect_class(class, name);
332    }
333
334    fn collect_class(&mut self, class: &Class<'_>, class_name: &str) {
335        for element in &class.body.body {
336            let ClassElement::MethodDefinition(method) = element else {
337                continue;
338            };
339            self.classified_spans
340                .insert((method.value.span.start, method.value.span.end));
341            if method.value.body.is_none() {
342                self.record_skip(SimilarCodeExtractionSkipReason::DeclarationWithoutBody, 1);
343                continue;
344            }
345            if method.kind != MethodDefinitionKind::Method || method.computed {
346                self.record_skip(SimilarCodeExtractionSkipReason::UnsupportedFunctionForm, 1);
347                continue;
348            }
349            let Some(method_name) = method.key.static_name() else {
350                self.record_skip(SimilarCodeExtractionSkipReason::UnsupportedFunctionForm, 1);
351                continue;
352            };
353            self.retain_function(
354                &format!("{class_name}.{method_name}"),
355                SimilarCodeFunctionKind::ClassMethod,
356                method.span,
357                ReviewMetadata::for_function(&method.value, self.syntax_reliable),
358            );
359        }
360    }
361
362    fn collect_object_methods(&mut self, object: &ObjectExpression<'_>, binding_name: &str) {
363        for property in &object.properties {
364            let ObjectPropertyKind::ObjectProperty(property) = property else {
365                continue;
366            };
367            if !property.method {
368                continue;
369            }
370            let oxc_ast::ast::Expression::FunctionExpression(function) = &property.value else {
371                continue;
372            };
373            self.classified_spans
374                .insert((function.span.start, function.span.end));
375            if function.body.is_none() {
376                self.record_skip(SimilarCodeExtractionSkipReason::DeclarationWithoutBody, 1);
377                continue;
378            }
379            if property.kind != PropertyKind::Init || property.computed {
380                self.record_skip(SimilarCodeExtractionSkipReason::UnsupportedFunctionForm, 1);
381                continue;
382            }
383            let Some(method_name) = property.key.static_name() else {
384                self.record_skip(SimilarCodeExtractionSkipReason::UnsupportedFunctionForm, 1);
385                continue;
386            };
387            self.retain_function(
388                &format!("{binding_name}.{method_name}"),
389                SimilarCodeFunctionKind::ObjectMethod,
390                property.span,
391                ReviewMetadata::for_function(function, self.syntax_reliable),
392            );
393        }
394    }
395
396    fn collect_function_declaration(&mut self, function: &Function<'_>, fallback: Option<&str>) {
397        self.classified_spans
398            .insert((function.span.start, function.span.end));
399        let Some(_body) = &function.body else {
400            self.record_skip(SimilarCodeExtractionSkipReason::DeclarationWithoutBody, 1);
401            return;
402        };
403        let Some(name) = function
404            .id
405            .as_ref()
406            .map(|identifier| identifier.name.as_str())
407            .or(fallback)
408        else {
409            self.record_skip(SimilarCodeExtractionSkipReason::UnsupportedFunctionForm, 1);
410            return;
411        };
412        self.retain_function(
413            name,
414            SimilarCodeFunctionKind::FunctionDeclaration,
415            function.span,
416            ReviewMetadata::for_function(function, self.syntax_reliable),
417        );
418    }
419
420    fn collect_function_expression(&mut self, function: &Function<'_>, name: &str) {
421        self.classified_spans
422            .insert((function.span.start, function.span.end));
423        if function.body.is_none() {
424            self.record_skip(SimilarCodeExtractionSkipReason::DeclarationWithoutBody, 1);
425            return;
426        }
427        self.retain_function(
428            name,
429            SimilarCodeFunctionKind::FunctionExpression,
430            function.span,
431            ReviewMetadata::for_function(function, self.syntax_reliable),
432        );
433    }
434
435    fn collect_arrow_function(&mut self, arrow: &ArrowFunctionExpression<'_>, name: &str) {
436        self.classified_spans
437            .insert((arrow.span.start, arrow.span.end));
438        self.retain_function(
439            name,
440            SimilarCodeFunctionKind::ArrowFunction,
441            arrow.span,
442            ReviewMetadata::for_arrow(arrow, self.syntax_reliable),
443        );
444    }
445
446    fn retain_function(
447        &mut self,
448        name: &str,
449        kind: SimilarCodeFunctionKind,
450        span: Span,
451        metadata: ReviewMetadata,
452    ) {
453        let start = span.start as usize;
454        let end = span.end as usize;
455        let Some(source) = self.source.get(start..end) else {
456            self.record_skip(SimilarCodeExtractionSkipReason::InvalidSourceSpan, 1);
457            return;
458        };
459        if source.len() > self.limits.max_source_bytes_per_function {
460            self.record_skip(
461                SimilarCodeExtractionSkipReason::SourceBytesPerFunctionLimit,
462                1,
463            );
464            return;
465        }
466        if self.functions.len() >= self.limits.max_functions {
467            self.record_skip(SimilarCodeExtractionSkipReason::FunctionLimit, 1);
468            return;
469        }
470        if self.source_bytes.saturating_add(source.len()) > self.limits.max_total_source_bytes {
471            self.record_skip(SimilarCodeExtractionSkipReason::TotalSourceBytesLimit, 1);
472            return;
473        }
474        let Some((start_line, start_column_utf8)) =
475            utf8_line_col(self.source, &self.line_offsets, span.start)
476        else {
477            self.record_skip(SimilarCodeExtractionSkipReason::InvalidSourceSpan, 1);
478            return;
479        };
480        let Some((end_line, end_column_utf8)) =
481            utf8_line_col(self.source, &self.line_offsets, span.end)
482        else {
483            self.record_skip(SimilarCodeExtractionSkipReason::InvalidSourceSpan, 1);
484            return;
485        };
486
487        let source_sha256 = SimilarCodeSourceDigest::new(Sha256::digest(source.as_bytes()).into());
488        self.source_bytes = self.source_bytes.saturating_add(source.len());
489        self.functions.push(ExtractedSimilarCodeFunction {
490            name: name.to_string(),
491            kind,
492            location: SimilarCodeFunctionLocation {
493                file: self.file.clone(),
494                start_byte: span.start,
495                end_byte: span.end,
496                start_line,
497                start_column_utf8,
498                end_line,
499                end_column_utf8,
500            },
501            source_sha256,
502            source: source.to_string(),
503            param_count: metadata.param_count,
504            is_async: metadata.is_async,
505            is_generator: metadata.is_generator,
506            has_await: metadata.has_await,
507            has_throw: metadata.has_throw,
508            side_effect_hint: metadata.side_effect_hint,
509        });
510    }
511
512    fn record_skip(&mut self, reason: SimilarCodeExtractionSkipReason, count: usize) {
513        let value = self.skips.entry(reason).or_default();
514        *value = value.saturating_add(count);
515    }
516
517    fn finish(mut self, program: &Program<'_>) -> SimilarCodeExtraction {
518        let mut visitor = UnsupportedFunctionVisitor::new(&self.classified_spans);
519        visitor.visit_program(program);
520        for (reason, count) in visitor.skips {
521            self.record_skip(reason, count);
522        }
523        build_result(self.functions, self.source_bytes, self.skips)
524    }
525}
526
527struct UnsupportedFunctionVisitor<'a> {
528    classified_spans: &'a BTreeSet<(u32, u32)>,
529    function_depth: usize,
530    skips: FxHashMap<SimilarCodeExtractionSkipReason, usize>,
531}
532
533impl<'a> UnsupportedFunctionVisitor<'a> {
534    fn new(classified_spans: &'a BTreeSet<(u32, u32)>) -> Self {
535        Self {
536            classified_spans,
537            function_depth: 0,
538            skips: FxHashMap::default(),
539        }
540    }
541
542    fn record_unclassified(&mut self, span: Span, has_body: bool) {
543        if self.classified_spans.contains(&(span.start, span.end)) {
544            return;
545        }
546        let reason = if !has_body {
547            SimilarCodeExtractionSkipReason::DeclarationWithoutBody
548        } else if self.function_depth > 0 {
549            SimilarCodeExtractionSkipReason::NestedFunction
550        } else {
551            SimilarCodeExtractionSkipReason::UnsupportedFunctionForm
552        };
553        let value = self.skips.entry(reason).or_default();
554        *value = value.saturating_add(1);
555    }
556}
557
558impl<'ast> Visit<'ast> for UnsupportedFunctionVisitor<'_> {
559    fn visit_function(&mut self, function: &Function<'ast>, flags: ScopeFlags) {
560        self.record_unclassified(function.span, function.body.is_some());
561        self.function_depth = self.function_depth.saturating_add(1);
562        walk::walk_function(self, function, flags);
563        self.function_depth = self.function_depth.saturating_sub(1);
564    }
565
566    fn visit_arrow_function_expression(&mut self, arrow: &ArrowFunctionExpression<'ast>) {
567        self.record_unclassified(arrow.span, true);
568        self.function_depth = self.function_depth.saturating_add(1);
569        walk::walk_arrow_function_expression(self, arrow);
570        self.function_depth = self.function_depth.saturating_sub(1);
571    }
572}
573
574/// Extract supported named top-level functions from one standalone JS or TS source.
575///
576/// Supported forms are named function declarations, simple identifier bindings
577/// initialized with a function expression or arrow, and equivalent default
578/// exports. Statically named ordinary methods on top-level named classes and
579/// bound object literals are also supported. Nested functions, callbacks,
580/// constructors, accessors, computed methods, declaration files, and generated
581/// sources are omitted with typed skip evidence. Source fragments are exact
582/// post-BOM UTF-8 slices and are bounded before hashing or allocation.
583#[must_use]
584pub fn extract_similar_code_functions(
585    path: &Path,
586    source: &str,
587    limits: SimilarCodeExtractionLimits,
588) -> SimilarCodeExtraction {
589    let Some(file) = path.to_str() else {
590        return single_skip_result(SimilarCodeExtractionSkipReason::NonUtf8Path);
591    };
592    let file = file.replace('\\', "/");
593    if is_declaration_file(&file) {
594        return single_skip_result(SimilarCodeExtractionSkipReason::DeclarationFile);
595    }
596    let Some(source_type) = supported_source_type(path) else {
597        return single_skip_result(SimilarCodeExtractionSkipReason::UnsupportedFileType);
598    };
599
600    let source = crate::strip_bom(source);
601    if is_generated_source(&file, source) {
602        return single_skip_result(SimilarCodeExtractionSkipReason::GeneratedSource);
603    }
604
605    let allocator = Allocator::default();
606    let parsed = Parser::new(&allocator, source, source_type).parse();
607    let syntax_reliable = parsed.errors.is_empty();
608    let mut builder = ExtractionBuilder::new(file, source, limits, syntax_reliable);
609    builder.record_skip(
610        SimilarCodeExtractionSkipReason::SyntaxDiagnostic,
611        parsed.errors.len(),
612    );
613    builder.collect_program(&parsed.program);
614    builder.finish(&parsed.program)
615}
616
617fn supported_source_type(path: &Path) -> Option<SourceType> {
618    let extension = path.extension()?.to_str()?.to_ascii_lowercase();
619    matches!(
620        extension.as_str(),
621        "js" | "jsx" | "mjs" | "cjs" | "ts" | "tsx" | "mts" | "cts"
622    )
623    .then(|| SourceType::from_path(path).unwrap_or_default())
624}
625
626fn is_declaration_file(file: &str) -> bool {
627    let file = file.to_ascii_lowercase();
628    file.ends_with(".d.ts") || file.ends_with(".d.mts") || file.ends_with(".d.cts")
629}
630
631fn is_generated_source(file: &str, source: &str) -> bool {
632    let lower_file = file.to_ascii_lowercase();
633    let generated_path = lower_file.split('/').any(|part| {
634        matches!(part, "generated" | "__generated__")
635            || part.contains(".generated.")
636            || part.contains(".gen.")
637    });
638    if generated_path {
639        return true;
640    }
641
642    let header = source
643        .chars()
644        .take(2_048)
645        .collect::<String>()
646        .to_ascii_lowercase();
647    header.contains("@generated")
648        || (header.contains("do not edit")
649            && (header.contains("code generated") || header.contains("automatically generated")))
650}
651
652fn utf8_line_col(source: &str, line_offsets: &[u32], byte_offset: u32) -> Option<(u32, u32)> {
653    let line_index = match line_offsets.binary_search(&byte_offset) {
654        Ok(index) => index,
655        Err(index) => index.saturating_sub(1),
656    };
657    let line_start = *line_offsets.get(line_index)? as usize;
658    let column = source
659        .get(line_start..byte_offset as usize)?
660        .chars()
661        .count();
662    Some((
663        u32::try_from(line_index).ok()?.saturating_add(1),
664        u32::try_from(column).ok()?,
665    ))
666}
667
668fn single_skip_result(reason: SimilarCodeExtractionSkipReason) -> SimilarCodeExtraction {
669    build_result(Vec::new(), 0, FxHashMap::from_iter([(reason, 1)]))
670}
671
672fn build_result(
673    functions: Vec<ExtractedSimilarCodeFunction>,
674    source_bytes: usize,
675    skips: FxHashMap<SimilarCodeExtractionSkipReason, usize>,
676) -> SimilarCodeExtraction {
677    let mut skipped = skips
678        .into_iter()
679        .filter(|(_, count)| *count > 0)
680        .map(|(reason, count)| SimilarCodeExtractionSkip { reason, count })
681        .collect::<Vec<_>>();
682    skipped.sort_by_key(|skip| skip.reason);
683    SimilarCodeExtraction {
684        extraction_semantics_version: SIMILAR_CODE_EXTRACTION_SEMANTICS_VERSION,
685        functions,
686        source_bytes,
687        skipped,
688    }
689}
690
691#[cfg(test)]
692mod tests {
693    use super::*;
694
695    fn extract(file: &str, source: &str) -> SimilarCodeExtraction {
696        extract_similar_code_functions(
697            Path::new(file),
698            source,
699            SimilarCodeExtractionLimits::default(),
700        )
701    }
702
703    fn skip_count(
704        result: &SimilarCodeExtraction,
705        reason: SimilarCodeExtractionSkipReason,
706    ) -> usize {
707        result
708            .skipped
709            .iter()
710            .find(|skip| skip.reason == reason)
711            .map_or(0, |skip| skip.count)
712    }
713
714    #[test]
715    fn extracts_supported_top_level_forms_in_source_order() {
716        let source = r"
717function declared(value: number) { return value + 1; }
718const expression = function named(value: number) { return value + 2; };
719export const arrow = (value: number) => value + 3;
720export default function fallback(value: number) { return value + 4; }
721";
722        let result = extract("src/forms.ts", source);
723
724        assert_eq!(
725            result
726                .functions
727                .iter()
728                .map(|function| (function.name.as_str(), function.kind))
729                .collect::<Vec<_>>(),
730            vec![
731                ("declared", SimilarCodeFunctionKind::FunctionDeclaration),
732                ("expression", SimilarCodeFunctionKind::FunctionExpression),
733                ("arrow", SimilarCodeFunctionKind::ArrowFunction),
734                ("fallback", SimilarCodeFunctionKind::FunctionDeclaration),
735            ]
736        );
737        assert!(result.skipped.is_empty());
738        assert_eq!(
739            result.source_bytes,
740            result
741                .functions
742                .iter()
743                .map(|function| function.source.len())
744                .sum::<usize>()
745        );
746    }
747
748    #[test]
749    fn records_exact_utf8_span_source_and_full_sha256() {
750        let source = "const label = '🙂'; const naïve = (waarde: string) => {\n  return `${label}:${waarde}`;\n};\n";
751        let result = extract("src/utf8.ts", source);
752        let function = &result.functions[0];
753        let expected_start = source.find("(waarde").unwrap();
754        let expected_end = source.rfind("};\n").map_or(source.len(), |index| index + 1);
755        let expected_source = &source[expected_start..expected_end];
756
757        assert_eq!(function.name, "naïve");
758        assert_eq!(function.location.start_byte as usize, expected_start);
759        assert_eq!(
760            function.location.start_column_utf8 as usize,
761            source[..expected_start].chars().count()
762        );
763        assert_eq!(function.source, expected_source);
764        assert_eq!(
765            function.source_sha256,
766            SimilarCodeSourceDigest::new(Sha256::digest(expected_source.as_bytes()).into())
767        );
768    }
769
770    #[test]
771    fn extracts_named_methods_and_skips_nested_or_unsupported_methods() {
772        let source = r#"
773function outer() {
774  const nested = () => 1;
775  return nested();
776}
777class Service {
778  constructor() {}
779  get value() { return 1; }
780  set value(next) {}
781  ["computed"]() {}
782  method(value) { return value + 2; }
783  static helper() { return 3; }
784}
785const object = {
786  get value() { return 1; },
787  ["computed"]() {},
788  method(value) { return value + 3; },
789};
790"#;
791        let result = extract("src/scopes.ts", source);
792
793        assert_eq!(
794            result
795                .functions
796                .iter()
797                .map(|function| (function.name.as_str(), function.kind))
798                .collect::<Vec<_>>(),
799            vec![
800                ("outer", SimilarCodeFunctionKind::FunctionDeclaration),
801                ("Service.method", SimilarCodeFunctionKind::ClassMethod),
802                ("Service.helper", SimilarCodeFunctionKind::ClassMethod),
803                ("object.method", SimilarCodeFunctionKind::ObjectMethod),
804            ]
805        );
806        assert_eq!(
807            skip_count(&result, SimilarCodeExtractionSkipReason::NestedFunction),
808            1
809        );
810        assert_eq!(
811            skip_count(
812                &result,
813                SimilarCodeExtractionSkipReason::UnsupportedFunctionForm
814            ),
815            6
816        );
817    }
818
819    #[test]
820    fn records_conservative_review_metadata_without_nested_leakage() {
821        let source = r#"
822async function load(input, ...rest) {
823  await fetch(input);
824  if (rest.length > 0) throw new Error("unexpected");
825}
826function* generate(value) { yield value; }
827function pure(left, right) { return left + right; }
828function wrapper() {
829  const nested = async () => { await fetch("nested"); throw new Error("nested"); };
830  return 1;
831}
832"#;
833        let result = extract("src/metadata.ts", source);
834        let find = |name: &str| {
835            result
836                .functions
837                .iter()
838                .find(|function| function.name == name)
839                .unwrap()
840        };
841
842        let load = find("load");
843        assert_eq!(load.param_count, 2);
844        assert!(load.is_async);
845        assert!(!load.is_generator);
846        assert!(load.has_await);
847        assert!(load.has_throw);
848        assert_eq!(
849            load.side_effect_hint,
850            SimilarCodeSideEffectHint::MayHaveSideEffects
851        );
852
853        let generate = find("generate");
854        assert!(generate.is_generator);
855        assert_eq!(
856            generate.side_effect_hint,
857            SimilarCodeSideEffectHint::MayHaveSideEffects
858        );
859
860        let pure = find("pure");
861        assert_eq!(pure.param_count, 2);
862        assert_eq!(
863            pure.side_effect_hint,
864            SimilarCodeSideEffectHint::PureLooking
865        );
866
867        let wrapper = find("wrapper");
868        assert!(!wrapper.has_await);
869        assert!(!wrapper.has_throw);
870        assert_eq!(
871            wrapper.side_effect_hint,
872            SimilarCodeSideEffectHint::PureLooking
873        );
874    }
875
876    #[test]
877    fn parser_recovery_marks_review_metadata_unknown() {
878        let result = extract(
879            "src/recovered.ts",
880            "function recovered(value) { return value; }\nreturn;",
881        );
882
883        assert_eq!(result.functions.len(), 1);
884        assert_eq!(
885            result.functions[0].side_effect_hint,
886            SimilarCodeSideEffectHint::Unknown
887        );
888        assert!(skip_count(&result, SimilarCodeExtractionSkipReason::SyntaxDiagnostic) > 0);
889    }
890
891    #[test]
892    fn generated_and_declaration_sources_are_excluded_before_parsing() {
893        let generated = extract(
894            "src/__generated__/client.ts",
895            "export const call = () => 1;",
896        );
897        assert_eq!(
898            generated.skipped,
899            vec![SimilarCodeExtractionSkip {
900                reason: SimilarCodeExtractionSkipReason::GeneratedSource,
901                count: 1,
902            }]
903        );
904
905        let declaration = extract("src/api.d.ts", "export declare function call(): void;");
906        assert_eq!(
907            declaration.skipped,
908            vec![SimilarCodeExtractionSkip {
909                reason: SimilarCodeExtractionSkipReason::DeclarationFile,
910                count: 1,
911            }]
912        );
913    }
914
915    #[test]
916    fn source_payload_limits_fail_closed_with_typed_counts() {
917        let source = "const first = () => 'this payload is definitely too large';\nconst second = () => 2;\nconst third = () => 3;\n";
918        let result = extract_similar_code_functions(
919            Path::new("src/limits.ts"),
920            source,
921            SimilarCodeExtractionLimits {
922                max_functions: 1,
923                max_source_bytes_per_function: 22,
924                max_total_source_bytes: 22,
925            },
926        );
927
928        assert_eq!(result.functions.len(), 1);
929        assert_eq!(
930            skip_count(
931                &result,
932                SimilarCodeExtractionSkipReason::SourceBytesPerFunctionLimit
933            ),
934            1
935        );
936        assert_eq!(
937            skip_count(&result, SimilarCodeExtractionSkipReason::FunctionLimit),
938            1
939        );
940        assert!(result.source_bytes <= 22);
941    }
942}
fallow_extract/similar_code.rs

fallow_extract/
similar_code.rs
