code_system_graph_core/
builtin_extractors.rs

1use std::time::Instant;
2
3use async_trait::async_trait;
4use code_system_graph_model::ArtifactFingerprint;
5use semver::Version;
6use serde::Serialize;
7
8use crate::{
9    BoundaryExtractor, ContentFingerprint, DiscoverContext, DiscoveredInput, ExtractInput, ExtractionBatch, ExtractionBudgets, ExtractionCompleteness, ExtractionReport, ExtractionTracker, ExtractorError, FileDescriptor, SourceEpistemicStatus, SourceObservation, SourceSyntaxLanguage, extract_generated_client_metadata, extract_package_manifest_with_tracker, inspect_source_syntax, parse_go_source_with_tracker, parse_java_source_with_tracker, parse_javascript_source_at_path_with_tracker, parse_python_source_with_tracker, parse_rust_source_with_tracker, parse_typescript_source_at_path_with_tracker
10};
11
12/// Focused source language selected for HTTP and test extraction.
13#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14pub enum FocusedSourceLanguage {
15    /// JavaScript framework syntax.
16    JavaScript,
17    /// TypeScript framework syntax.
18    TypeScript,
19    /// Rust framework syntax.
20    Rust,
21    /// Python framework syntax.
22    Python,
23    /// Go framework syntax.
24    Go,
25    /// Java framework syntax.
26    Java,
27}
28
29/// Built-in focused source HTTP and test extractor.
30pub struct FocusedSourceExtractor {
31    language: FocusedSourceLanguage,
32    budgets: ExtractionBudgets,
33}
34
35impl FocusedSourceExtractor {
36    /// Creates an extractor for one mandatory source-language matrix.
37    #[must_use]
38    pub fn new(language: FocusedSourceLanguage) -> Self {
39        Self::with_budgets(language, ExtractionBudgets::default())
40    }
41
42    /// Creates an extractor with explicit effective per-invocation budgets.
43    #[must_use]
44    pub fn with_budgets(language: FocusedSourceLanguage, budgets: ExtractionBudgets) -> Self {
45        Self { language, budgets }
46    }
47}
48
49#[async_trait]
50impl BoundaryExtractor for FocusedSourceExtractor {
51    fn id(&self) -> &'static str {
52        match self.language {
53            FocusedSourceLanguage::JavaScript => "code-system-graph.source.javascript",
54            FocusedSourceLanguage::TypeScript => "code-system-graph.source.typescript",
55            FocusedSourceLanguage::Rust => "code-system-graph.source.rust",
56            FocusedSourceLanguage::Python => "code-system-graph.source.python",
57            FocusedSourceLanguage::Go => "code-system-graph.source.go",
58            FocusedSourceLanguage::Java => "code-system-graph.source.java",
59        }
60    }
61
62    fn version(&self) -> Version {
63        Version::new(1, 0, 0)
64    }
65
66    fn supports(&self, file: &FileDescriptor) -> bool {
67        let extension = file
68            .path
69            .display
70            .rsplit_once('.')
71            .map(|(_, extension)| extension);
72        matches!(
73            (self.language, extension),
74            (FocusedSourceLanguage::JavaScript, Some("js" | "jsx"))
75                | (FocusedSourceLanguage::TypeScript, Some("ts" | "tsx"))
76                | (FocusedSourceLanguage::Rust, Some("rs"))
77                | (FocusedSourceLanguage::Python, Some("py"))
78                | (FocusedSourceLanguage::Go, Some("go"))
79                | (FocusedSourceLanguage::Java, Some("java"))
80        )
81    }
82
83    async fn discover(
84        &self,
85        context: &DiscoverContext<'_>,
86    ) -> Result<Vec<DiscoveredInput>, ExtractorError> {
87        Ok(context
88            .files
89            .iter()
90            .filter(|file| self.supports(file))
91            .cloned()
92            .map(|file| DiscoveredInput { file })
93            .collect())
94    }
95
96    async fn extract(&self, input: &ExtractInput<'_>) -> Result<ExtractionBatch, ExtractorError> {
97        if !self.supports(input.file) {
98            return Err(ExtractorError::InvalidInput(format!(
99                "{} does not support `{}`",
100                self.id(),
101                input.file.path.display
102            )));
103        }
104        let started = Instant::now();
105        let mut tracker =
106            crate::ExtractionTracker::new(&input.file.path.display, self.id(), &self.budgets);
107        tracker.check_input_bytes(u64::try_from(input.content.len()).unwrap_or(u64::MAX))?;
108        let fingerprint = fingerprint_with_budgets(input, self.id(), &self.budgets)?;
109        let source = std::str::from_utf8(input.content)?;
110        let syntax = inspect_source_syntax(
111            syntax_language(self.language),
112            &input.file.path.display,
113            source,
114            &mut tracker,
115        )
116        .map_err(extractor_source_syntax_error)?;
117        let reserved_observations = u64::try_from(syntax.boundary_candidate_count)
118            .map_err(|_| ExtractorError::InvalidInput("too many syntax candidates".to_owned()))?;
119        tracker.charge_work(reserved_observations)?;
120        precheck_focused_source_values(source, &mut tracker)?;
121        let observations = match self.language {
122            FocusedSourceLanguage::JavaScript => parse_javascript_source_at_path_with_tracker(
123                &input.file.path.display,
124                source,
125                &mut tracker,
126            )?,
127            FocusedSourceLanguage::TypeScript => parse_typescript_source_at_path_with_tracker(
128                &input.file.path.display,
129                source,
130                &mut tracker,
131            )?,
132            FocusedSourceLanguage::Rust => parse_rust_source_with_tracker(source, &mut tracker)?,
133            FocusedSourceLanguage::Python => {
134                parse_python_source_with_tracker(source, &mut tracker)?
135            }
136            FocusedSourceLanguage::Go => parse_go_source_with_tracker(source, &mut tracker)?,
137            FocusedSourceLanguage::Java => parse_java_source_with_tracker(source, &mut tracker)?,
138        };
139        if !observations.is_empty() && syntax.boundary_candidate_count == 0 {
140            return Err(ExtractorError::InvalidInput(
141                "framework observation lacked a Tree-sitter boundary candidate".to_owned(),
142            ));
143        }
144        let completeness = if !syntax.has_error
145            && observations
146                .iter()
147                .all(|observation| observation.status == SourceEpistemicStatus::Confirmed)
148        {
149            ExtractionCompleteness::Complete
150        } else {
151            ExtractionCompleteness::Partial
152        };
153        let mut warnings = observations
154            .iter()
155            .flat_map(|observation| &observation.warnings)
156            .map(|warning| format!("{warning:?}"))
157            .collect::<Vec<_>>();
158        if syntax.has_error {
159            warnings.push("Tree-sitter recovered from source syntax errors".to_owned());
160        }
161        let output_count = u64::try_from(observations.len())
162            .map_err(|_| ExtractorError::InvalidInput("too many observations".to_owned()))?;
163        let payload = serialize_bounded(&observations, &tracker)?;
164        tracker.check_structured_time()?;
165        Ok(ExtractionBatch {
166            source: ArtifactFingerprint {
167                repo_id: input.file.repo_id.clone(),
168                checkout_id: input.file.checkout_id.clone(),
169                path: input.file.path.clone(),
170                extractor: self.id().to_owned(),
171                content_hash: fingerprint.content_hash,
172                size_bytes: fingerprint.size_bytes,
173            },
174            payload,
175            output_count,
176            report: ExtractionReport {
177                discovered_files: 1,
178                parsed_files: 1,
179                skipped_files: 0,
180                completeness,
181                warnings,
182                evidence_count: output_count,
183                extractor_version: self.version(),
184                elapsed_ms: elapsed_ms(started),
185            },
186        })
187    }
188
189    async fn fingerprint(
190        &self,
191        input: &ExtractInput<'_>,
192    ) -> Result<crate::ContentFingerprint, ExtractorError> {
193        fingerprint_with_budgets(input, self.id(), &self.budgets)
194    }
195}
196
197fn syntax_language(language: FocusedSourceLanguage) -> SourceSyntaxLanguage {
198    match language {
199        FocusedSourceLanguage::JavaScript => SourceSyntaxLanguage::JavaScript,
200        FocusedSourceLanguage::TypeScript => SourceSyntaxLanguage::TypeScript,
201        FocusedSourceLanguage::Rust => SourceSyntaxLanguage::Rust,
202        FocusedSourceLanguage::Python => SourceSyntaxLanguage::Python,
203        FocusedSourceLanguage::Go => SourceSyntaxLanguage::Go,
204        FocusedSourceLanguage::Java => SourceSyntaxLanguage::Java,
205    }
206}
207
208/// Preflights source tokens before focused parsers allocate owned observation fields.
209///
210/// # Errors
211///
212/// Returns [`crate::ExtractionLimitExceeded`] when work, value, accumulated-string, or time
213/// budgets are exceeded.
214#[doc(hidden)]
215pub fn precheck_focused_source_values(
216    source: &str,
217    tracker: &mut ExtractionTracker,
218) -> Result<(), crate::ExtractionLimitExceeded> {
219    let bytes = source.as_bytes();
220    let mut cursor = 0_usize;
221    let mut accumulated = 0_u64;
222    while cursor < bytes.len() {
223        if cursor.is_multiple_of(1_024) {
224            tracker.check_structured_time()?;
225        }
226        let byte = bytes[cursor];
227        if matches!(byte, b'"' | b'\'' | b'`') {
228            tracker.charge_work(1)?;
229            let delimiter = byte;
230            cursor = cursor.saturating_add(1);
231            let start = cursor;
232            let mut escaped = false;
233            while cursor < bytes.len() {
234                let byte = bytes[cursor];
235                if escaped {
236                    escaped = false;
237                } else if byte == b'\\' {
238                    escaped = true;
239                } else if byte == delimiter {
240                    break;
241                }
242                cursor = cursor.saturating_add(1);
243                if cursor.is_multiple_of(1_024) {
244                    tracker.check_structured_time()?;
245                }
246            }
247            let observed = u64::try_from(cursor.saturating_sub(start)).unwrap_or(u64::MAX);
248            tracker.check_string_bytes(observed)?;
249            accumulated = accumulated.saturating_add(observed);
250            tracker.check_accumulated_string_bytes(accumulated)?;
251        } else if byte == b'_' || byte.is_ascii_alphabetic() {
252            tracker.charge_work(1)?;
253            let start = cursor;
254            cursor = cursor.saturating_add(1);
255            while cursor < bytes.len()
256                && (bytes[cursor] == b'_' || bytes[cursor].is_ascii_alphanumeric())
257            {
258                cursor = cursor.saturating_add(1);
259            }
260            let observed = u64::try_from(cursor.saturating_sub(start)).unwrap_or(u64::MAX);
261            tracker.check_identifier_bytes(observed)?;
262            accumulated = accumulated.saturating_add(observed);
263            tracker.check_accumulated_string_bytes(accumulated)?;
264            continue;
265        }
266        cursor = cursor.saturating_add(1);
267    }
268    Ok(())
269}
270
271/// Built-in `OpenAPI` Generator metadata extractor.
272pub struct GeneratedClientMetadataExtractor {
273    budgets: ExtractionBudgets,
274}
275
276impl GeneratedClientMetadataExtractor {
277    /// Creates an extractor with safe default budgets.
278    #[must_use]
279    pub fn new() -> Self {
280        Self::with_budgets(ExtractionBudgets::default())
281    }
282
283    /// Creates an extractor with explicit effective per-invocation budgets.
284    #[must_use]
285    pub const fn with_budgets(budgets: ExtractionBudgets) -> Self {
286        Self { budgets }
287    }
288}
289
290impl Default for GeneratedClientMetadataExtractor {
291    fn default() -> Self {
292        Self::new()
293    }
294}
295
296#[async_trait]
297impl BoundaryExtractor for GeneratedClientMetadataExtractor {
298    fn id(&self) -> &'static str {
299        "code-system-graph.http.generated-client"
300    }
301
302    fn version(&self) -> Version {
303        Version::new(1, 0, 0)
304    }
305
306    fn supports(&self, file: &FileDescriptor) -> bool {
307        generated_client_path_supported(&file.path.display)
308    }
309
310    async fn discover(
311        &self,
312        context: &DiscoverContext<'_>,
313    ) -> Result<Vec<DiscoveredInput>, ExtractorError> {
314        Ok(context
315            .files
316            .iter()
317            .filter(|file| self.supports(file))
318            .cloned()
319            .map(|file| DiscoveredInput { file })
320            .collect())
321    }
322
323    async fn extract(&self, input: &ExtractInput<'_>) -> Result<ExtractionBatch, ExtractorError> {
324        if !self.supports(input.file) {
325            return Err(ExtractorError::InvalidInput(format!(
326                "{} does not support `{}`",
327                self.id(),
328                input.file.path.display
329            )));
330        }
331        let started = Instant::now();
332        let mut tracker =
333            crate::ExtractionTracker::new(&input.file.path.display, self.id(), &self.budgets);
334        tracker.check_input_bytes(u64::try_from(input.content.len()).unwrap_or(u64::MAX))?;
335        let fingerprint = fingerprint_with_budgets(input, self.id(), &self.budgets)?;
336        let source = std::str::from_utf8(input.content)?;
337        let metadata =
338            extract_generated_client_metadata(&input.file.path.display, source, &mut tracker)
339                .map_err(|error| match error {
340                    crate::GeneratedClientError::LimitExceeded(limit) => {
341                        ExtractorError::LimitExceeded(limit)
342                    }
343                    error => ExtractorError::InvalidInput(error.to_string()),
344                })?;
345        let output_count = u64::try_from(metadata.len())
346            .map_err(|_| ExtractorError::InvalidInput("too many metadata facts".to_owned()))?;
347        let payload = serialize_bounded(&metadata, &tracker)?;
348        tracker.check_structured_time()?;
349        Ok(ExtractionBatch {
350            source: ArtifactFingerprint {
351                repo_id: input.file.repo_id.clone(),
352                checkout_id: input.file.checkout_id.clone(),
353                path: input.file.path.clone(),
354                extractor: self.id().to_owned(),
355                content_hash: fingerprint.content_hash,
356                size_bytes: fingerprint.size_bytes,
357            },
358            payload,
359            output_count,
360            report: ExtractionReport {
361                discovered_files: 1,
362                parsed_files: 1,
363                skipped_files: 0,
364                completeness: ExtractionCompleteness::Complete,
365                warnings: Vec::new(),
366                evidence_count: output_count,
367                extractor_version: self.version(),
368                elapsed_ms: elapsed_ms(started),
369            },
370        })
371    }
372
373    async fn fingerprint(
374        &self,
375        input: &ExtractInput<'_>,
376    ) -> Result<crate::ContentFingerprint, ExtractorError> {
377        fingerprint_with_budgets(input, self.id(), &self.budgets)
378    }
379}
380
381/// Built-in package-manifest extractor for the mandatory package ecosystems.
382pub struct PackageManifestExtractor {
383    budgets: ExtractionBudgets,
384}
385
386impl PackageManifestExtractor {
387    /// Creates an extractor with safe default budgets.
388    #[must_use]
389    pub fn new() -> Self {
390        Self::with_budgets(ExtractionBudgets::default())
391    }
392
393    /// Creates an extractor with explicit effective per-invocation budgets.
394    #[must_use]
395    pub const fn with_budgets(budgets: ExtractionBudgets) -> Self {
396        Self { budgets }
397    }
398}
399
400impl Default for PackageManifestExtractor {
401    fn default() -> Self {
402        Self::new()
403    }
404}
405
406#[async_trait]
407impl BoundaryExtractor for PackageManifestExtractor {
408    fn id(&self) -> &'static str {
409        "code-system-graph.packages"
410    }
411
412    fn version(&self) -> Version {
413        Version::new(1, 0, 0)
414    }
415
416    fn supports(&self, file: &FileDescriptor) -> bool {
417        package_path_supported(&file.path.display)
418    }
419
420    async fn discover(
421        &self,
422        context: &DiscoverContext<'_>,
423    ) -> Result<Vec<DiscoveredInput>, ExtractorError> {
424        Ok(context
425            .files
426            .iter()
427            .filter(|file| self.supports(file))
428            .cloned()
429            .map(|file| DiscoveredInput { file })
430            .collect())
431    }
432
433    async fn extract(&self, input: &ExtractInput<'_>) -> Result<ExtractionBatch, ExtractorError> {
434        if !self.supports(input.file) {
435            return Err(ExtractorError::InvalidInput(format!(
436                "{} does not support `{}`",
437                self.id(),
438                input.file.path.display
439            )));
440        }
441        let started = Instant::now();
442        let mut tracker =
443            crate::ExtractionTracker::new(&input.file.path.display, self.id(), &self.budgets);
444        tracker.check_input_bytes(u64::try_from(input.content.len()).unwrap_or(u64::MAX))?;
445        let fingerprint = fingerprint_with_budgets(input, self.id(), &self.budgets)?;
446        let source = std::str::from_utf8(input.content)?;
447        let manifest =
448            extract_package_manifest_with_tracker(&input.file.path.display, source, &mut tracker)
449                .map_err(|error| match error {
450                crate::PackageManifestError::LimitExceeded(limit) => {
451                    ExtractorError::LimitExceeded(limit)
452                }
453                error => ExtractorError::InvalidInput(error.to_string()),
454            })?;
455        let output_count = [
456            manifest.packages.len(),
457            manifest.dependencies.len(),
458            manifest.workspace_members.len(),
459            manifest.exports.len(),
460            manifest.features.len(),
461            manifest.lockfiles.len(),
462        ]
463        .into_iter()
464        .try_fold(0_u64, |total, count| {
465            u64::try_from(count)
466                .ok()
467                .and_then(|count| total.checked_add(count))
468        })
469        .ok_or_else(|| ExtractorError::InvalidInput("too many package facts".to_owned()))?;
470        let payload = serialize_bounded(&manifest, &tracker)?;
471        tracker.check_structured_time()?;
472        Ok(ExtractionBatch {
473            source: ArtifactFingerprint {
474                repo_id: input.file.repo_id.clone(),
475                checkout_id: input.file.checkout_id.clone(),
476                path: input.file.path.clone(),
477                extractor: self.id().to_owned(),
478                content_hash: fingerprint.content_hash,
479                size_bytes: fingerprint.size_bytes,
480            },
481            payload,
482            output_count,
483            report: ExtractionReport {
484                discovered_files: 1,
485                parsed_files: 1,
486                skipped_files: 0,
487                completeness: ExtractionCompleteness::Complete,
488                warnings: Vec::new(),
489                evidence_count: output_count,
490                extractor_version: self.version(),
491                elapsed_ms: elapsed_ms(started),
492            },
493        })
494    }
495
496    async fn fingerprint(
497        &self,
498        input: &ExtractInput<'_>,
499    ) -> Result<crate::ContentFingerprint, ExtractorError> {
500        fingerprint_with_budgets(input, self.id(), &self.budgets)
501    }
502}
503
504fn extractor_source_syntax_error(error: crate::SourceSyntaxError) -> ExtractorError {
505    match error {
506        crate::SourceSyntaxError::LimitExceeded(limit) => ExtractorError::LimitExceeded(limit),
507        error => ExtractorError::InvalidInput(error.to_string()),
508    }
509}
510
511fn fingerprint_with_budgets(
512    input: &ExtractInput<'_>,
513    extractor: &str,
514    budgets: &ExtractionBudgets,
515) -> Result<ContentFingerprint, ExtractorError> {
516    let observed = u64::try_from(input.content.len()).unwrap_or(u64::MAX);
517    ExtractionTracker::new(&input.file.path.display, extractor, budgets)
518        .check_input_bytes(observed)?;
519    Ok(ContentFingerprint {
520        content_hash: blake3::hash(input.content).to_hex().to_string(),
521        size_bytes: observed,
522    })
523}
524
525/// Charges every retained value in one focused source observation.
526///
527/// # Errors
528///
529/// Returns [`crate::ExtractionLimitExceeded`] before retaining values beyond effective budgets.
530#[doc(hidden)]
531pub fn charge_source_observation(
532    observation: &SourceObservation,
533    tracker: &mut ExtractionTracker,
534) -> Result<(), crate::ExtractionLimitExceeded> {
535    crate::source_http::charge_source_observation_values(observation, tracker)
536}
537
538fn serialize_bounded<T: Serialize>(
539    value: &T,
540    tracker: &ExtractionTracker,
541) -> Result<Vec<u8>, ExtractorError> {
542    let mut writer = tracker.bounded_json_writer();
543    if let Err(error) = serde_json::to_writer(&mut writer, value) {
544        if let Some(limit) = tracker.output_limit_error(&writer) {
545            return Err(ExtractorError::LimitExceeded(limit));
546        }
547        return Err(ExtractorError::InvalidOutput(error));
548    }
549    Ok(writer.into_inner())
550}
551
552fn package_path_supported(path: &str) -> bool {
553    let name = path.rsplit(['/', '\\']).next().unwrap_or(path);
554    matches!(
555        name,
556        "package.json"
557            | "package-lock.json"
558            | "npm-shrinkwrap.json"
559            | "pnpm-lock.yaml"
560            | "yarn.lock"
561            | "pyproject.toml"
562            | "Cargo.toml"
563            | "go.mod"
564            | "go.work"
565            | "pom.xml"
566            | "build.gradle"
567            | "build.gradle.kts"
568            | "packages.config"
569    ) || (name.starts_with("requirements")
570        && std::path::Path::new(name)
571            .extension()
572            .is_some_and(|extension| extension.eq_ignore_ascii_case("txt")))
573        || name.to_ascii_lowercase().ends_with(".csproj")
574}
575
576fn generated_client_path_supported(path: &str) -> bool {
577    let path = std::path::Path::new(path);
578    let Some(name) = path.file_name().and_then(std::ffi::OsStr::to_str) else {
579        return false;
580    };
581    name == "openapitools.json"
582        || (matches!(name, "FILES" | "VERSION")
583            && path
584                .parent()
585                .and_then(std::path::Path::file_name)
586                .is_some_and(|parent| parent == ".openapi-generator"))
587}
588
589fn elapsed_ms(started: Instant) -> u64 {
590    u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX)
591}
592
593#[cfg(test)]
594mod tests {
595    use code_system_graph_model::{CheckoutId, NativePath, NativePathEncoding, RepoId};
596
597    use super::{
598        BoundaryExtractor, ExtractInput, ExtractionBudgets, ExtractorError, FileDescriptor, FocusedSourceExtractor, FocusedSourceLanguage, GeneratedClientMetadataExtractor, PackageManifestExtractor
599    };
600
601    fn file(path: &str) -> FileDescriptor {
602        FileDescriptor {
603            repo_id: RepoId::new("repo:test"),
604            checkout_id: CheckoutId::new("checkout:test"),
605            path: NativePath {
606                encoding: NativePathEncoding::Utf8,
607                bytes: path.as_bytes().to_vec(),
608                display: path.to_owned(),
609            },
610            size_bytes: 1,
611        }
612    }
613
614    #[tokio::test]
615    async fn source_extractor_should_emit_versioned_json_without_source_text() {
616        let extractor = FocusedSourceExtractor::new(FocusedSourceLanguage::Rust);
617        let file = file("src/routes.rs");
618        let source =
619            b"use axum::{Router, routing::get}; Router::new().route(\"/health\", get(health));";
620        let result = extractor
621            .extract(&ExtractInput {
622                file: &file,
623                content: source,
624            })
625            .await;
626
627        assert!(matches!(
628            result,
629            Ok(batch)
630                if batch.output_count == 1
631                    && !String::from_utf8_lossy(&batch.payload).contains("Router::new")
632        ));
633    }
634
635    #[tokio::test]
636    async fn source_extractor_should_preflight_work_and_values() {
637        let file = file("src/routes.rs");
638        let source =
639            b"use axum::{Router, routing::get}; Router::new().route(\"/health\", get(health));";
640        let cases = [
641            (
642                ExtractionBudgets {
643                    max_work_units_per_artifact: 1,
644                    ..ExtractionBudgets::default()
645                },
646                crate::ExtractionResource::WorkUnits,
647            ),
648            (
649                ExtractionBudgets {
650                    max_string_bytes_per_value: 3,
651                    ..ExtractionBudgets::default()
652                },
653                crate::ExtractionResource::StringBytesPerValue,
654            ),
655        ];
656
657        for (budgets, resource) in cases {
658            let extractor =
659                FocusedSourceExtractor::with_budgets(FocusedSourceLanguage::Rust, budgets);
660            let result = extractor
661                .extract(&ExtractInput {
662                    file: &file,
663                    content: source,
664                })
665                .await;
666            assert!(matches!(
667                result,
668                Err(ExtractorError::LimitExceeded(error)) if error.resource == resource
669            ));
670        }
671    }
672
673    #[tokio::test]
674    async fn source_extractor_should_charge_each_observation_before_retention() {
675        let file = file("tests/test_api.py");
676        let source = b"import requests\ndef test_create_order():\n    requests.post(\"https://api.test/v1/orders\")\n";
677        let one = ExtractionBudgets {
678            max_observations_per_artifact: 1,
679            ..ExtractionBudgets::default()
680        };
681        let rejected = FocusedSourceExtractor::with_budgets(FocusedSourceLanguage::Python, one)
682            .extract(&ExtractInput {
683                file: &file,
684                content: source,
685            })
686            .await;
687        assert!(matches!(
688            rejected,
689            Err(ExtractorError::LimitExceeded(error))
690                if error.resource == crate::ExtractionResource::Observations
691                    && error.observed == 2
692                    && error.maximum == 1
693        ));
694
695        let exact = ExtractionBudgets {
696            max_observations_per_artifact: 2,
697            ..ExtractionBudgets::default()
698        };
699        let accepted = FocusedSourceExtractor::with_budgets(FocusedSourceLanguage::Python, exact)
700            .extract(&ExtractInput {
701                file: &file,
702                content: source,
703            })
704            .await;
705        assert!(matches!(accepted, Ok(batch) if batch.output_count == 2));
706    }
707
708    #[tokio::test]
709    async fn javascript_extractor_should_preserve_next_file_route_context() {
710        let extractor = FocusedSourceExtractor::new(FocusedSourceLanguage::JavaScript);
711        let file = file("frontend/src/app/api/logout/route.js");
712        let source = b"export async function GET() { return new Response(); }\n";
713        let batch = extractor
714            .extract(&ExtractInput {
715                file: &file,
716                content: source,
717            })
718            .await
719            .expect("Next.js route should extract");
720        let observations: Vec<crate::SourceObservation> =
721            serde_json::from_slice(&batch.payload).expect("valid observation payload");
722
723        assert!(observations.iter().any(|observation| {
724            observation.framework == crate::SourceFramework::NextJs
725                && observation.method.as_deref() == Some("GET")
726                && observation.path.as_deref() == Some("/api/logout")
727        }));
728    }
729
730    #[tokio::test]
731    async fn generated_client_extractor_should_emit_explicit_config_facts() {
732        let extractor = GeneratedClientMetadataExtractor::default();
733        let file = file("openapitools.json");
734        let result = extractor
735            .extract(&ExtractInput {
736                file: &file,
737                content: br#"{"generator-cli":{"generators":{"client":{"generatorName":"rust","inputSpec":"openapi.yaml"}}}}"#,
738            })
739            .await;
740
741        assert!(matches!(result, Ok(batch) if batch.output_count == 1));
742    }
743
744    #[tokio::test]
745    async fn package_extractor_should_omit_evidence_source_lines_from_payload() {
746        let extractor = PackageManifestExtractor::default();
747        let file = file("Cargo.toml");
748        let source = b"[package]\nname = \"api\"\nversion = \"1.0.0\"\n";
749        let result = extractor
750            .extract(&ExtractInput {
751                file: &file,
752                content: source,
753            })
754            .await;
755
756        assert!(matches!(
757            result,
758            Ok(batch)
759                if batch.output_count == 1
760                    && !String::from_utf8_lossy(&batch.payload).contains("name =")
761        ));
762    }
763
764    #[tokio::test]
765    async fn built_in_extractors_should_honor_explicit_effective_budgets() {
766        let budgets = ExtractionBudgets {
767            max_serialized_output_bytes_per_artifact: 1,
768            ..ExtractionBudgets::default()
769        };
770        let extractor = GeneratedClientMetadataExtractor::with_budgets(budgets);
771        let file = file("openapitools.json");
772        let result = extractor
773            .extract(&ExtractInput {
774                file: &file,
775                content: br#"{"generator-cli":{"generators":{"client":{"generatorName":"rust"}}}}"#,
776            })
777            .await;
778
779        assert!(matches!(
780            result,
781            Err(ExtractorError::LimitExceeded(error))
782                if error.resource
783                    == crate::ExtractionResource::SerializedOutputBytes
784                    && error.maximum == 1
785        ));
786    }
787}
code_system_graph_core/builtin_extractors.rs

code_system_graph_core/
builtin_extractors.rs
