fallow_extract/
similar_code.rs

1//! On-demand, bounded JS and TS function extraction for similar-code providers.
2
3use std::collections::BTreeSet;
4use std::path::Path;
5
6use fallow_types::similar_code::{
7    ExtractedSimilarCodeFunction, SIMILAR_CODE_EXTRACTION_SEMANTICS_VERSION, SimilarCodeExtraction,
8    SimilarCodeExtractionLimits, SimilarCodeExtractionSkip, SimilarCodeExtractionSkipReason,
9    SimilarCodeFunctionKind, SimilarCodeFunctionLocation, SimilarCodeSideEffectHint,
10    SimilarCodeSourceDigest,
11};
12use oxc_allocator::Allocator;
13use oxc_ast::ast::{
14    ArrowFunctionExpression, AssignmentExpression, AwaitExpression, CallExpression, Class,
15    ClassElement, ComputedMemberExpression, Declaration, ExportDefaultDeclarationKind, Function,
16    ImportExpression, JSXElement, JSXFragment, MethodDefinitionKind, NewExpression,
17    ObjectExpression, ObjectPropertyKind, PrivateFieldExpression, Program, PropertyKind, Statement,
18    StaticMemberExpression, TaggedTemplateExpression, ThrowStatement, UnaryExpression,
19    UpdateExpression, VariableDeclaration, YieldExpression,
20};
21use oxc_ast_visit::{Visit, walk};
22use oxc_parser::Parser;
23use oxc_semantic::ScopeFlags;
24use oxc_span::{GetSpan, SourceType, Span};
25use rustc_hash::FxHashMap;
26use sha2::{Digest, Sha256};
27
28use crate::function_body::BodyRef;
29use crate::js_text::JsText;
30
31#[derive(Debug, Clone, Copy)]
32struct ReviewMetadata {
33    param_count: u32,
34    is_async: bool,
35    is_generator: bool,
36    has_await: bool,
37    has_throw: bool,
38    side_effect_hint: SimilarCodeSideEffectHint,
39}
40
41impl ReviewMetadata {
42    fn for_function(function: &Function<'_>, syntax_reliable: bool) -> Self {
43        Self::from_body(
44            function.body.as_deref().map(BodyRef::Block),
45            function.params.items.len(),
46            function.params.rest.is_some(),
47            function.r#async,
48            function.generator,
49            syntax_reliable,
50        )
51    }
52
53    fn for_arrow(arrow: &ArrowFunctionExpression<'_>, syntax_reliable: bool) -> Self {
54        Self::from_body(
55            Some(BodyRef::arrow(&arrow.body)),
56            arrow.params.items.len(),
57            arrow.params.rest.is_some(),
58            arrow.r#async,
59            false,
60            syntax_reliable,
61        )
62    }
63
64    fn from_body(
65        body: Option<BodyRef<'_, '_>>,
66        fixed_params: usize,
67        has_rest: bool,
68        is_async: bool,
69        is_generator: bool,
70        syntax_reliable: bool,
71    ) -> Self {
72        let mut visitor = ReviewMetadataVisitor::default();
73        if let Some(body) = body {
74            body.visit(&mut visitor);
75        }
76        let param_count = fixed_params.saturating_add(usize::from(has_rest));
77        let side_effect_hint = if !syntax_reliable {
78            SimilarCodeSideEffectHint::Unknown
79        } else if visitor.may_have_side_effects {
80            SimilarCodeSideEffectHint::MayHaveSideEffects
81        } else {
82            SimilarCodeSideEffectHint::PureLooking
83        };
84        Self {
85            param_count: u32::try_from(param_count).unwrap_or(u32::MAX),
86            is_async,
87            is_generator,
88            has_await: visitor.has_await,
89            has_throw: visitor.has_throw,
90            side_effect_hint,
91        }
92    }
93}
94
95#[derive(Default)]
96struct ReviewMetadataVisitor {
97    has_await: bool,
98    has_throw: bool,
99    may_have_side_effects: bool,
100}
101
102impl<'ast> Visit<'ast> for ReviewMetadataVisitor {
103    fn visit_function(&mut self, _function: &Function<'ast>, _flags: ScopeFlags) {}
104
105    fn visit_arrow_function_expression(&mut self, _arrow: &ArrowFunctionExpression<'ast>) {}
106
107    fn visit_call_expression(&mut self, expression: &CallExpression<'ast>) {
108        self.may_have_side_effects = true;
109        walk::walk_call_expression(self, expression);
110    }
111
112    fn visit_new_expression(&mut self, expression: &NewExpression<'ast>) {
113        self.may_have_side_effects = true;
114        walk::walk_new_expression(self, expression);
115    }
116
117    fn visit_assignment_expression(&mut self, expression: &AssignmentExpression<'ast>) {
118        self.may_have_side_effects = true;
119        walk::walk_assignment_expression(self, expression);
120    }
121
122    fn visit_update_expression(&mut self, expression: &UpdateExpression<'ast>) {
123        self.may_have_side_effects = true;
124        walk::walk_update_expression(self, expression);
125    }
126
127    fn visit_unary_expression(&mut self, expression: &UnaryExpression<'ast>) {
128        self.may_have_side_effects |= expression.operator.is_delete();
129        walk::walk_unary_expression(self, expression);
130    }
131
132    fn visit_await_expression(&mut self, expression: &AwaitExpression<'ast>) {
133        self.has_await = true;
134        self.may_have_side_effects = true;
135        walk::walk_await_expression(self, expression);
136    }
137
138    fn visit_throw_statement(&mut self, statement: &ThrowStatement<'ast>) {
139        self.has_throw = true;
140        self.may_have_side_effects = true;
141        walk::walk_throw_statement(self, statement);
142    }
143
144    fn visit_yield_expression(&mut self, expression: &YieldExpression<'ast>) {
145        self.may_have_side_effects = true;
146        walk::walk_yield_expression(self, expression);
147    }
148
149    fn visit_import_expression(&mut self, expression: &ImportExpression<'ast>) {
150        self.may_have_side_effects = true;
151        walk::walk_import_expression(self, expression);
152    }
153
154    fn visit_tagged_template_expression(&mut self, expression: &TaggedTemplateExpression<'ast>) {
155        self.may_have_side_effects = true;
156        walk::walk_tagged_template_expression(self, expression);
157    }
158
159    fn visit_computed_member_expression(&mut self, expression: &ComputedMemberExpression<'ast>) {
160        self.may_have_side_effects = true;
161        walk::walk_computed_member_expression(self, expression);
162    }
163
164    fn visit_static_member_expression(&mut self, expression: &StaticMemberExpression<'ast>) {
165        self.may_have_side_effects = true;
166        walk::walk_static_member_expression(self, expression);
167    }
168
169    fn visit_private_field_expression(&mut self, expression: &PrivateFieldExpression<'ast>) {
170        self.may_have_side_effects = true;
171        walk::walk_private_field_expression(self, expression);
172    }
173
174    fn visit_jsx_element(&mut self, element: &JSXElement<'ast>) {
175        self.may_have_side_effects = true;
176        walk::walk_jsx_element(self, element);
177    }
178
179    fn visit_jsx_fragment(&mut self, fragment: &JSXFragment<'ast>) {
180        self.may_have_side_effects = true;
181        walk::walk_jsx_fragment(self, fragment);
182    }
183}
184
185struct ExtractionBuilder<'a> {
186    file: String,
187    source: &'a str,
188    line_offsets: Vec<u32>,
189    limits: SimilarCodeExtractionLimits,
190    functions: Vec<ExtractedSimilarCodeFunction>,
191    source_bytes: usize,
192    syntax_reliable: bool,
193    classified_spans: BTreeSet<(u32, u32)>,
194    skips: FxHashMap<SimilarCodeExtractionSkipReason, usize>,
195}
196
197impl<'a> ExtractionBuilder<'a> {
198    fn new(
199        file: String,
200        source: &'a str,
201        limits: SimilarCodeExtractionLimits,
202        syntax_reliable: bool,
203    ) -> Self {
204        Self {
205            file,
206            source,
207            line_offsets: fallow_types::extract::compute_line_offsets(source),
208            limits,
209            functions: Vec::new(),
210            source_bytes: 0,
211            syntax_reliable,
212            classified_spans: BTreeSet::new(),
213            skips: FxHashMap::default(),
214        }
215    }
216
217    fn collect_program(&mut self, program: &Program<'_>) {
218        for statement in &program.body {
219            match statement {
220                Statement::FunctionDeclaration(function) => {
221                    self.collect_function_declaration(function, None);
222                }
223                Statement::VariableDeclaration(declaration) => {
224                    self.collect_variable_declaration(declaration);
225                }
226                Statement::ClassDeclaration(class) => {
227                    self.collect_class_declaration(class, None);
228                }
229                Statement::ExportDeclaration(export) => {
230                    self.collect_declaration(&export.declaration);
231                }
232                Statement::ExportDefaultDeclaration(export) => {
233                    self.collect_default_export(&export.declaration);
234                }
235                _ => {}
236            }
237        }
238    }
239
240    fn collect_declaration(&mut self, declaration: &Declaration<'_>) {
241        match declaration {
242            Declaration::FunctionDeclaration(function) => {
243                self.collect_function_declaration(function, None);
244            }
245            Declaration::VariableDeclaration(declaration) => {
246                self.collect_variable_declaration(declaration);
247            }
248            Declaration::ClassDeclaration(class) => {
249                self.collect_class_declaration(class, None);
250            }
251            _ => {}
252        }
253    }
254
255    fn collect_default_export(&mut self, declaration: &ExportDefaultDeclarationKind<'_>) {
256        match declaration {
257            ExportDefaultDeclarationKind::FunctionDeclaration(function) => {
258                self.collect_function_declaration(function, Some("default"));
259            }
260            ExportDefaultDeclarationKind::FunctionExpression(function) => {
261                self.collect_function_expression(function, "default");
262            }
263            ExportDefaultDeclarationKind::ArrowFunctionExpression(arrow) => {
264                self.collect_arrow_function(arrow, "default");
265            }
266            ExportDefaultDeclarationKind::ClassDeclaration(class)
267            | ExportDefaultDeclarationKind::ClassExpression(class) => {
268                self.collect_class_declaration(class, Some("default"));
269            }
270            ExportDefaultDeclarationKind::ObjectExpression(object) => {
271                self.collect_object_methods(object, "default");
272            }
273            _ => {}
274        }
275    }
276
277    fn collect_variable_declaration(&mut self, declaration: &VariableDeclaration<'_>) {
278        for declarator in &declaration.declarations {
279            let Some(initializer) = &declarator.init else {
280                continue;
281            };
282            let is_supported_container = matches!(
283                initializer,
284                oxc_ast::ast::Expression::FunctionExpression(_)
285                    | oxc_ast::ast::Expression::ArrowFunctionExpression(_)
286                    | oxc_ast::ast::Expression::ClassExpression(_)
287                    | oxc_ast::ast::Expression::ObjectExpression(_)
288            );
289            if !is_supported_container {
290                continue;
291            }
292
293            let Some(binding) = declarator.id.get_binding_identifier() else {
294                if matches!(
295                    initializer,
296                    oxc_ast::ast::Expression::FunctionExpression(_)
297                        | oxc_ast::ast::Expression::ArrowFunctionExpression(_)
298                ) {
299                    self.classified_spans
300                        .insert((initializer.span().start, initializer.span().end));
301                    self.record_skip(SimilarCodeExtractionSkipReason::UnsupportedFunctionForm, 1);
302                }
303                continue;
304            };
305            match initializer {
306                oxc_ast::ast::Expression::FunctionExpression(function) => {
307                    self.collect_function_expression(function, binding.name.as_str());
308                }
309                oxc_ast::ast::Expression::ArrowFunctionExpression(arrow) => {
310                    self.collect_arrow_function(arrow, binding.name.as_str());
311                }
312                oxc_ast::ast::Expression::ClassExpression(class) => {
313                    self.collect_class(class, binding.name.as_str());
314                }
315                oxc_ast::ast::Expression::ObjectExpression(object) => {
316                    self.collect_object_methods(object, binding.name.as_str());
317                }
318                _ => {}
319            }
320        }
321    }
322
323    fn collect_class_declaration(&mut self, class: &Class<'_>, fallback: Option<&str>) {
324        let Some(name) = class
325            .id
326            .as_ref()
327            .map(|identifier| identifier.name.as_str())
328            .or(fallback)
329        else {
330            return;
331        };
332        self.collect_class(class, name);
333    }
334
335    fn collect_class(&mut self, class: &Class<'_>, class_name: &str) {
336        for element in &class.body.body {
337            let ClassElement::MethodDefinition(method) = element else {
338                continue;
339            };
340            self.classified_spans
341                .insert((method.value.span.start, method.value.span.end));
342            if method.value.body.is_none() {
343                self.record_skip(SimilarCodeExtractionSkipReason::DeclarationWithoutBody, 1);
344                continue;
345            }
346            if method.kind != MethodDefinitionKind::Method || method.computed {
347                self.record_skip(SimilarCodeExtractionSkipReason::UnsupportedFunctionForm, 1);
348                continue;
349            }
350            let Some(method_name) = method.key.static_name() else {
351                self.record_skip(SimilarCodeExtractionSkipReason::UnsupportedFunctionForm, 1);
352                continue;
353            };
354            self.retain_function(
355                &format!("{class_name}.{}", method_name.text()),
356                SimilarCodeFunctionKind::ClassMethod,
357                method.span,
358                ReviewMetadata::for_function(&method.value, self.syntax_reliable),
359            );
360        }
361    }
362
363    fn collect_object_methods(&mut self, object: &ObjectExpression<'_>, binding_name: &str) {
364        for property in &object.properties {
365            let ObjectPropertyKind::ObjectProperty(property) = property else {
366                continue;
367            };
368            if !property.method {
369                continue;
370            }
371            let oxc_ast::ast::Expression::FunctionExpression(function) = &property.value else {
372                continue;
373            };
374            self.classified_spans
375                .insert((function.span.start, function.span.end));
376            if function.body.is_none() {
377                self.record_skip(SimilarCodeExtractionSkipReason::DeclarationWithoutBody, 1);
378                continue;
379            }
380            if property.kind != PropertyKind::Init || property.computed {
381                self.record_skip(SimilarCodeExtractionSkipReason::UnsupportedFunctionForm, 1);
382                continue;
383            }
384            let Some(method_name) = property.key.static_name() else {
385                self.record_skip(SimilarCodeExtractionSkipReason::UnsupportedFunctionForm, 1);
386                continue;
387            };
388            self.retain_function(
389                &format!("{binding_name}.{}", method_name.text()),
390                SimilarCodeFunctionKind::ObjectMethod,
391                property.span,
392                ReviewMetadata::for_function(function, self.syntax_reliable),
393            );
394        }
395    }
396
397    fn collect_function_declaration(&mut self, function: &Function<'_>, fallback: Option<&str>) {
398        self.classified_spans
399            .insert((function.span.start, function.span.end));
400        let Some(_body) = &function.body else {
401            self.record_skip(SimilarCodeExtractionSkipReason::DeclarationWithoutBody, 1);
402            return;
403        };
404        let Some(name) = function
405            .id
406            .as_ref()
407            .map(|identifier| identifier.name.as_str())
408            .or(fallback)
409        else {
410            self.record_skip(SimilarCodeExtractionSkipReason::UnsupportedFunctionForm, 1);
411            return;
412        };
413        self.retain_function(
414            name,
415            SimilarCodeFunctionKind::FunctionDeclaration,
416            function.span,
417            ReviewMetadata::for_function(function, self.syntax_reliable),
418        );
419    }
420
421    fn collect_function_expression(&mut self, function: &Function<'_>, name: &str) {
422        self.classified_spans
423            .insert((function.span.start, function.span.end));
424        if function.body.is_none() {
425            self.record_skip(SimilarCodeExtractionSkipReason::DeclarationWithoutBody, 1);
426            return;
427        }
428        self.retain_function(
429            name,
430            SimilarCodeFunctionKind::FunctionExpression,
431            function.span,
432            ReviewMetadata::for_function(function, self.syntax_reliable),
433        );
434    }
435
436    fn collect_arrow_function(&mut self, arrow: &ArrowFunctionExpression<'_>, name: &str) {
437        self.classified_spans
438            .insert((arrow.span.start, arrow.span.end));
439        self.retain_function(
440            name,
441            SimilarCodeFunctionKind::ArrowFunction,
442            arrow.span,
443            ReviewMetadata::for_arrow(arrow, self.syntax_reliable),
444        );
445    }
446
447    fn retain_function(
448        &mut self,
449        name: &str,
450        kind: SimilarCodeFunctionKind,
451        span: Span,
452        metadata: ReviewMetadata,
453    ) {
454        let start = span.start as usize;
455        let end = span.end as usize;
456        let Some(source) = self.source.get(start..end) else {
457            self.record_skip(SimilarCodeExtractionSkipReason::InvalidSourceSpan, 1);
458            return;
459        };
460        if source.len() > self.limits.max_source_bytes_per_function {
461            self.record_skip(
462                SimilarCodeExtractionSkipReason::SourceBytesPerFunctionLimit,
463                1,
464            );
465            return;
466        }
467        if self.functions.len() >= self.limits.max_functions {
468            self.record_skip(SimilarCodeExtractionSkipReason::FunctionLimit, 1);
469            return;
470        }
471        if self.source_bytes.saturating_add(source.len()) > self.limits.max_total_source_bytes {
472            self.record_skip(SimilarCodeExtractionSkipReason::TotalSourceBytesLimit, 1);
473            return;
474        }
475        let Some((start_line, start_column_utf8)) =
476            utf8_line_col(self.source, &self.line_offsets, span.start)
477        else {
478            self.record_skip(SimilarCodeExtractionSkipReason::InvalidSourceSpan, 1);
479            return;
480        };
481        let Some((end_line, end_column_utf8)) =
482            utf8_line_col(self.source, &self.line_offsets, span.end)
483        else {
484            self.record_skip(SimilarCodeExtractionSkipReason::InvalidSourceSpan, 1);
485            return;
486        };
487
488        let source_sha256 = SimilarCodeSourceDigest::new(Sha256::digest(source.as_bytes()).into());
489        self.source_bytes = self.source_bytes.saturating_add(source.len());
490        self.functions.push(ExtractedSimilarCodeFunction {
491            name: name.to_string(),
492            kind,
493            location: SimilarCodeFunctionLocation {
494                file: self.file.clone(),
495                start_byte: span.start,
496                end_byte: span.end,
497                start_line,
498                start_column_utf8,
499                end_line,
500                end_column_utf8,
501            },
502            source_sha256,
503            source: source.to_string(),
504            param_count: metadata.param_count,
505            is_async: metadata.is_async,
506            is_generator: metadata.is_generator,
507            has_await: metadata.has_await,
508            has_throw: metadata.has_throw,
509            side_effect_hint: metadata.side_effect_hint,
510        });
511    }
512
513    fn record_skip(&mut self, reason: SimilarCodeExtractionSkipReason, count: usize) {
514        let value = self.skips.entry(reason).or_default();
515        *value = value.saturating_add(count);
516    }
517
518    fn finish(mut self, program: &Program<'_>) -> SimilarCodeExtraction {
519        let mut visitor = UnsupportedFunctionVisitor::new(&self.classified_spans);
520        visitor.visit_program(program);
521        for (reason, count) in visitor.skips {
522            self.record_skip(reason, count);
523        }
524        build_result(self.functions, self.source_bytes, self.skips)
525    }
526}
527
528struct UnsupportedFunctionVisitor<'a> {
529    classified_spans: &'a BTreeSet<(u32, u32)>,
530    function_depth: usize,
531    skips: FxHashMap<SimilarCodeExtractionSkipReason, usize>,
532}
533
534impl<'a> UnsupportedFunctionVisitor<'a> {
535    fn new(classified_spans: &'a BTreeSet<(u32, u32)>) -> Self {
536        Self {
537            classified_spans,
538            function_depth: 0,
539            skips: FxHashMap::default(),
540        }
541    }
542
543    fn record_unclassified(&mut self, span: Span, has_body: bool) {
544        if self.classified_spans.contains(&(span.start, span.end)) {
545            return;
546        }
547        let reason = if !has_body {
548            SimilarCodeExtractionSkipReason::DeclarationWithoutBody
549        } else if self.function_depth > 0 {
550            SimilarCodeExtractionSkipReason::NestedFunction
551        } else {
552            SimilarCodeExtractionSkipReason::UnsupportedFunctionForm
553        };
554        let value = self.skips.entry(reason).or_default();
555        *value = value.saturating_add(1);
556    }
557}
558
559impl<'ast> Visit<'ast> for UnsupportedFunctionVisitor<'_> {
560    fn visit_function(&mut self, function: &Function<'ast>, flags: ScopeFlags) {
561        self.record_unclassified(function.span, function.body.is_some());
562        self.function_depth = self.function_depth.saturating_add(1);
563        walk::walk_function(self, function, flags);
564        self.function_depth = self.function_depth.saturating_sub(1);
565    }
566
567    fn visit_arrow_function_expression(&mut self, arrow: &ArrowFunctionExpression<'ast>) {
568        self.record_unclassified(arrow.span, true);
569        self.function_depth = self.function_depth.saturating_add(1);
570        walk::walk_arrow_function_expression(self, arrow);
571        self.function_depth = self.function_depth.saturating_sub(1);
572    }
573}
574
575/// Extract supported named top-level functions from one standalone JS or TS source.
576///
577/// Supported forms are named function declarations, simple identifier bindings
578/// initialized with a function expression or arrow, and equivalent default
579/// exports. Statically named ordinary methods on top-level named classes and
580/// bound object literals are also supported. Nested functions, callbacks,
581/// constructors, accessors, computed methods, declaration files, and generated
582/// sources are omitted with typed skip evidence. Source fragments are exact
583/// post-BOM UTF-8 slices and are bounded before hashing or allocation.
584#[must_use]
585pub fn extract_similar_code_functions(
586    path: &Path,
587    source: &str,
588    limits: SimilarCodeExtractionLimits,
589) -> SimilarCodeExtraction {
590    let Some(file) = path.to_str() else {
591        return single_skip_result(SimilarCodeExtractionSkipReason::NonUtf8Path);
592    };
593    let file = file.replace('\\', "/");
594    if is_declaration_file(&file) {
595        return single_skip_result(SimilarCodeExtractionSkipReason::DeclarationFile);
596    }
597    let Some(source_type) = supported_source_type(path) else {
598        return single_skip_result(SimilarCodeExtractionSkipReason::UnsupportedFileType);
599    };
600
601    let source = crate::strip_bom(source);
602    if is_generated_source(&file, source) {
603        return single_skip_result(SimilarCodeExtractionSkipReason::GeneratedSource);
604    }
605    crate::parse_bounded(source, || {
606        let allocator = Allocator::default();
607        let parsed = Parser::new(&allocator, source, source_type).parse();
608        let syntax_reliable = parsed.diagnostics.is_empty();
609        let mut builder = ExtractionBuilder::new(file, source, limits, syntax_reliable);
610        builder.record_skip(
611            SimilarCodeExtractionSkipReason::SyntaxDiagnostic,
612            parsed.diagnostics.len(),
613        );
614        builder.collect_program(&parsed.program);
615        builder.finish(&parsed.program)
616    })
617    .unwrap_or_else(|| single_skip_result(SimilarCodeExtractionSkipReason::NestingLimit))
618}
619
620fn supported_source_type(path: &Path) -> Option<SourceType> {
621    let extension = path.extension()?.to_str()?.to_ascii_lowercase();
622    matches!(
623        extension.as_str(),
624        "js" | "jsx" | "mjs" | "cjs" | "ts" | "tsx" | "mts" | "cts"
625    )
626    .then(|| SourceType::from_path(path).unwrap_or_default())
627}
628
629fn is_declaration_file(file: &str) -> bool {
630    let file = file.to_ascii_lowercase();
631    file.ends_with(".d.ts") || file.ends_with(".d.mts") || file.ends_with(".d.cts")
632}
633
634fn is_generated_source(file: &str, source: &str) -> bool {
635    let lower_file = file.to_ascii_lowercase();
636    let generated_path = lower_file.split('/').any(|part| {
637        matches!(part, "generated" | "__generated__")
638            || part.contains(".generated.")
639            || part.contains(".gen.")
640    });
641    if generated_path {
642        return true;
643    }
644
645    let header = source
646        .chars()
647        .take(2_048)
648        .collect::<String>()
649        .to_ascii_lowercase();
650    header.contains("@generated")
651        || (header.contains("do not edit")
652            && (header.contains("code generated") || header.contains("automatically generated")))
653}
654
655fn utf8_line_col(source: &str, line_offsets: &[u32], byte_offset: u32) -> Option<(u32, u32)> {
656    let line_index = match line_offsets.binary_search(&byte_offset) {
657        Ok(index) => index,
658        Err(index) => index.saturating_sub(1),
659    };
660    let line_start = *line_offsets.get(line_index)? as usize;
661    let column = source
662        .get(line_start..byte_offset as usize)?
663        .chars()
664        .count();
665    Some((
666        u32::try_from(line_index).ok()?.saturating_add(1),
667        u32::try_from(column).ok()?,
668    ))
669}
670
671fn single_skip_result(reason: SimilarCodeExtractionSkipReason) -> SimilarCodeExtraction {
672    build_result(Vec::new(), 0, FxHashMap::from_iter([(reason, 1)]))
673}
674
675fn build_result(
676    functions: Vec<ExtractedSimilarCodeFunction>,
677    source_bytes: usize,
678    skips: FxHashMap<SimilarCodeExtractionSkipReason, usize>,
679) -> SimilarCodeExtraction {
680    let mut skipped = skips
681        .into_iter()
682        .filter(|(_, count)| *count > 0)
683        .map(|(reason, count)| SimilarCodeExtractionSkip { reason, count })
684        .collect::<Vec<_>>();
685    skipped.sort_by_key(|skip| skip.reason);
686    SimilarCodeExtraction {
687        extraction_semantics_version: SIMILAR_CODE_EXTRACTION_SEMANTICS_VERSION,
688        functions,
689        source_bytes,
690        skipped,
691    }
692}
693
694#[cfg(test)]
695mod tests {
696    use super::*;
697
698    fn extract(file: &str, source: &str) -> SimilarCodeExtraction {
699        extract_similar_code_functions(
700            Path::new(file),
701            source,
702            SimilarCodeExtractionLimits::default(),
703        )
704    }
705
706    fn skip_count(
707        result: &SimilarCodeExtraction,
708        reason: SimilarCodeExtractionSkipReason,
709    ) -> usize {
710        result
711            .skipped
712            .iter()
713            .find(|skip| skip.reason == reason)
714            .map_or(0, |skip| skip.count)
715    }
716
717    #[test]
718    fn extracts_supported_top_level_forms_in_source_order() {
719        let source = r"
720function declared(value: number) { return value + 1; }
721const expression = function named(value: number) { return value + 2; };
722export const arrow = (value: number) => value + 3;
723export default function fallback(value: number) { return value + 4; }
724";
725        let result = extract("src/forms.ts", source);
726
727        assert_eq!(
728            result
729                .functions
730                .iter()
731                .map(|function| (function.name.as_str(), function.kind))
732                .collect::<Vec<_>>(),
733            vec![
734                ("declared", SimilarCodeFunctionKind::FunctionDeclaration),
735                ("expression", SimilarCodeFunctionKind::FunctionExpression),
736                ("arrow", SimilarCodeFunctionKind::ArrowFunction),
737                ("fallback", SimilarCodeFunctionKind::FunctionDeclaration),
738            ]
739        );
740        assert!(result.skipped.is_empty());
741        assert_eq!(
742            result.source_bytes,
743            result
744                .functions
745                .iter()
746                .map(|function| function.source.len())
747                .sum::<usize>()
748        );
749    }
750
751    #[test]
752    fn records_exact_utf8_span_source_and_full_sha256() {
753        let source = "const label = '🙂'; const naïve = (waarde: string) => {\n  return `${label}:${waarde}`;\n};\n";
754        let result = extract("src/utf8.ts", source);
755        let function = &result.functions[0];
756        let expected_start = source.find("(waarde").unwrap();
757        let expected_end = source.rfind("};\n").map_or(source.len(), |index| index + 1);
758        let expected_source = &source[expected_start..expected_end];
759
760        assert_eq!(function.name, "naïve");
761        assert_eq!(function.location.start_byte as usize, expected_start);
762        assert_eq!(
763            function.location.start_column_utf8 as usize,
764            source[..expected_start].chars().count()
765        );
766        assert_eq!(function.source, expected_source);
767        assert_eq!(
768            function.source_sha256,
769            SimilarCodeSourceDigest::new(Sha256::digest(expected_source.as_bytes()).into())
770        );
771    }
772
773    #[test]
774    fn extracts_named_methods_and_skips_nested_or_unsupported_methods() {
775        let source = r#"
776function outer() {
777  const nested = () => 1;
778  return nested();
779}
780class Service {
781  constructor() {}
782  get value() { return 1; }
783  set value(next) {}
784  ["computed"]() {}
785  method(value) { return value + 2; }
786  static helper() { return 3; }
787}
788const object = {
789  get value() { return 1; },
790  ["computed"]() {},
791  method(value) { return value + 3; },
792};
793"#;
794        let result = extract("src/scopes.ts", source);
795
796        assert_eq!(
797            result
798                .functions
799                .iter()
800                .map(|function| (function.name.as_str(), function.kind))
801                .collect::<Vec<_>>(),
802            vec![
803                ("outer", SimilarCodeFunctionKind::FunctionDeclaration),
804                ("Service.method", SimilarCodeFunctionKind::ClassMethod),
805                ("Service.helper", SimilarCodeFunctionKind::ClassMethod),
806                ("object.method", SimilarCodeFunctionKind::ObjectMethod),
807            ]
808        );
809        assert_eq!(
810            skip_count(&result, SimilarCodeExtractionSkipReason::NestedFunction),
811            1
812        );
813        assert_eq!(
814            skip_count(
815                &result,
816                SimilarCodeExtractionSkipReason::UnsupportedFunctionForm
817            ),
818            6
819        );
820    }
821
822    #[test]
823    fn records_conservative_review_metadata_without_nested_leakage() {
824        let source = r#"
825async function load(input, ...rest) {
826  await fetch(input);
827  if (rest.length > 0) throw new Error("unexpected");
828}
829function* generate(value) { yield value; }
830function pure(left, right) { return left + right; }
831function wrapper() {
832  const nested = async () => { await fetch("nested"); throw new Error("nested"); };
833  return 1;
834}
835"#;
836        let result = extract("src/metadata.ts", source);
837        let find = |name: &str| {
838            result
839                .functions
840                .iter()
841                .find(|function| function.name == name)
842                .unwrap()
843        };
844
845        let load = find("load");
846        assert_eq!(load.param_count, 2);
847        assert!(load.is_async);
848        assert!(!load.is_generator);
849        assert!(load.has_await);
850        assert!(load.has_throw);
851        assert_eq!(
852            load.side_effect_hint,
853            SimilarCodeSideEffectHint::MayHaveSideEffects
854        );
855
856        let generate = find("generate");
857        assert!(generate.is_generator);
858        assert_eq!(
859            generate.side_effect_hint,
860            SimilarCodeSideEffectHint::MayHaveSideEffects
861        );
862
863        let pure = find("pure");
864        assert_eq!(pure.param_count, 2);
865        assert_eq!(
866            pure.side_effect_hint,
867            SimilarCodeSideEffectHint::PureLooking
868        );
869
870        let wrapper = find("wrapper");
871        assert!(!wrapper.has_await);
872        assert!(!wrapper.has_throw);
873        assert_eq!(
874            wrapper.side_effect_hint,
875            SimilarCodeSideEffectHint::PureLooking
876        );
877    }
878
879    #[test]
880    fn parser_recovery_marks_review_metadata_unknown() {
881        let result = extract(
882            "src/recovered.ts",
883            "function recovered(value) { return value; }\nreturn;",
884        );
885
886        assert_eq!(result.functions.len(), 1);
887        assert_eq!(
888            result.functions[0].side_effect_hint,
889            SimilarCodeSideEffectHint::Unknown
890        );
891        assert!(skip_count(&result, SimilarCodeExtractionSkipReason::SyntaxDiagnostic) > 0);
892    }
893
894    #[test]
895    fn generated_and_declaration_sources_are_excluded_before_parsing() {
896        let generated = extract(
897            "src/__generated__/client.ts",
898            "export const call = () => 1;",
899        );
900        assert_eq!(
901            generated.skipped,
902            vec![SimilarCodeExtractionSkip {
903                reason: SimilarCodeExtractionSkipReason::GeneratedSource,
904                count: 1,
905            }]
906        );
907
908        let declaration = extract("src/api.d.ts", "export declare function call(): void;");
909        assert_eq!(
910            declaration.skipped,
911            vec![SimilarCodeExtractionSkip {
912                reason: SimilarCodeExtractionSkipReason::DeclarationFile,
913                count: 1,
914            }]
915        );
916    }
917
918    #[test]
919    fn source_payload_limits_fail_closed_with_typed_counts() {
920        let source = "const first = () => 'this payload is definitely too large';\nconst second = () => 2;\nconst third = () => 3;\n";
921        let result = extract_similar_code_functions(
922            Path::new("src/limits.ts"),
923            source,
924            SimilarCodeExtractionLimits {
925                max_functions: 1,
926                max_source_bytes_per_function: 22,
927                max_total_source_bytes: 22,
928            },
929        );
930
931        assert_eq!(result.functions.len(), 1);
932        assert_eq!(
933            skip_count(
934                &result,
935                SimilarCodeExtractionSkipReason::SourceBytesPerFunctionLimit
936            ),
937            1
938        );
939        assert_eq!(
940            skip_count(&result, SimilarCodeExtractionSkipReason::FunctionLimit),
941            1
942        );
943        assert!(result.source_bytes <= 22);
944    }
945}
fallow_extract/similar_code.rs

fallow_extract/
similar_code.rs
