Skip to main content

code_system_graph_core/
builtin_extractors.rs

1use std::time::Instant;
2
3use async_trait::async_trait;
4use code_system_graph_model::ArtifactFingerprint;
5use semver::Version;
6use serde::Serialize;
7
8use crate::{
9    BoundaryExtractor, ContentFingerprint, DiscoverContext, DiscoveredInput, ExtractInput, ExtractionBatch, ExtractionBudgets, ExtractionCompleteness, ExtractionReport, ExtractionTracker, ExtractorError, FileDescriptor, SourceEpistemicStatus, SourceObservation, SourceSyntaxLanguage, extract_generated_client_metadata, extract_package_manifest_with_tracker, inspect_source_syntax, parse_go_source_with_tracker, parse_java_source_with_tracker, parse_javascript_source_at_path_with_tracker, parse_python_source_with_tracker, parse_rust_source_with_tracker, parse_typescript_source_at_path_with_tracker
10};
11
12/// Focused source language selected for HTTP and test extraction.
13#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14pub enum FocusedSourceLanguage {
15    /// JavaScript framework syntax.
16    JavaScript,
17    /// TypeScript framework syntax.
18    TypeScript,
19    /// Rust framework syntax.
20    Rust,
21    /// Python framework syntax.
22    Python,
23    /// Go framework syntax.
24    Go,
25    /// Java framework syntax.
26    Java,
27}
28
29/// Built-in focused source HTTP and test extractor.
30pub struct FocusedSourceExtractor {
31    language: FocusedSourceLanguage,
32    budgets: ExtractionBudgets,
33}
34
35impl FocusedSourceExtractor {
36    /// Creates an extractor for one mandatory source-language matrix.
37    #[must_use]
38    pub fn new(language: FocusedSourceLanguage) -> Self {
39        Self::with_budgets(language, ExtractionBudgets::default())
40    }
41
42    /// Creates an extractor with explicit effective per-invocation budgets.
43    #[must_use]
44    pub fn with_budgets(language: FocusedSourceLanguage, budgets: ExtractionBudgets) -> Self {
45        Self { language, budgets }
46    }
47}
48
49#[async_trait]
50impl BoundaryExtractor for FocusedSourceExtractor {
51    fn id(&self) -> &'static str {
52        match self.language {
53            FocusedSourceLanguage::JavaScript => "code-system-graph.source.javascript",
54            FocusedSourceLanguage::TypeScript => "code-system-graph.source.typescript",
55            FocusedSourceLanguage::Rust => "code-system-graph.source.rust",
56            FocusedSourceLanguage::Python => "code-system-graph.source.python",
57            FocusedSourceLanguage::Go => "code-system-graph.source.go",
58            FocusedSourceLanguage::Java => "code-system-graph.source.java",
59        }
60    }
61
62    fn version(&self) -> Version {
63        Version::new(1, 0, 0)
64    }
65
66    fn supports(&self, file: &FileDescriptor) -> bool {
67        let extension = file
68            .path
69            .display
70            .rsplit_once('.')
71            .map(|(_, extension)| extension);
72        matches!(
73            (self.language, extension),
74            (FocusedSourceLanguage::JavaScript, Some("js" | "jsx"))
75                | (FocusedSourceLanguage::TypeScript, Some("ts" | "tsx"))
76                | (FocusedSourceLanguage::Rust, Some("rs"))
77                | (FocusedSourceLanguage::Python, Some("py"))
78                | (FocusedSourceLanguage::Go, Some("go"))
79                | (FocusedSourceLanguage::Java, Some("java"))
80        )
81    }
82
83    async fn discover(
84        &self,
85        context: &DiscoverContext<'_>,
86    ) -> Result<Vec<DiscoveredInput>, ExtractorError> {
87        Ok(context
88            .files
89            .iter()
90            .filter(|file| self.supports(file))
91            .cloned()
92            .map(|file| DiscoveredInput { file })
93            .collect())
94    }
95
96    async fn extract(&self, input: &ExtractInput<'_>) -> Result<ExtractionBatch, ExtractorError> {
97        if !self.supports(input.file) {
98            return Err(ExtractorError::InvalidInput(format!(
99                "{} does not support `{}`",
100                self.id(),
101                input.file.path.display
102            )));
103        }
104        let started = Instant::now();
105        let mut tracker =
106            crate::ExtractionTracker::new(&input.file.path.display, self.id(), &self.budgets);
107        tracker.check_input_bytes(u64::try_from(input.content.len()).unwrap_or(u64::MAX))?;
108        let fingerprint = fingerprint_with_budgets(input, self.id(), &self.budgets)?;
109        let source = std::str::from_utf8(input.content)?;
110        let syntax = inspect_source_syntax(
111            syntax_language(self.language),
112            &input.file.path.display,
113            source,
114            &mut tracker,
115        )
116        .map_err(extractor_source_syntax_error)?;
117        let reserved_observations = u64::try_from(syntax.boundary_candidate_count)
118            .map_err(|_| ExtractorError::InvalidInput("too many syntax candidates".to_owned()))?;
119        tracker.charge_work(reserved_observations)?;
120        precheck_focused_source_values(source, syntax_language(self.language), &mut tracker)?;
121        let observations = match self.language {
122            FocusedSourceLanguage::JavaScript => parse_javascript_source_at_path_with_tracker(
123                &input.file.path.display,
124                source,
125                &mut tracker,
126            )?,
127            FocusedSourceLanguage::TypeScript => parse_typescript_source_at_path_with_tracker(
128                &input.file.path.display,
129                source,
130                &mut tracker,
131            )?,
132            FocusedSourceLanguage::Rust => parse_rust_source_with_tracker(source, &mut tracker)?,
133            FocusedSourceLanguage::Python => {
134                parse_python_source_with_tracker(source, &mut tracker)?
135            }
136            FocusedSourceLanguage::Go => parse_go_source_with_tracker(source, &mut tracker)?,
137            FocusedSourceLanguage::Java => parse_java_source_with_tracker(source, &mut tracker)?,
138        };
139        if !observations.is_empty() && syntax.boundary_candidate_count == 0 {
140            return Err(ExtractorError::InvalidInput(
141                "framework observation lacked a Tree-sitter boundary candidate".to_owned(),
142            ));
143        }
144        let completeness = if !syntax.has_error
145            && observations
146                .iter()
147                .all(|observation| observation.status == SourceEpistemicStatus::Confirmed)
148        {
149            ExtractionCompleteness::Complete
150        } else {
151            ExtractionCompleteness::Partial
152        };
153        let mut warnings = observations
154            .iter()
155            .flat_map(|observation| &observation.warnings)
156            .map(|warning| format!("{warning:?}"))
157            .collect::<Vec<_>>();
158        if syntax.has_error {
159            warnings.push("Tree-sitter recovered from source syntax errors".to_owned());
160        }
161        let output_count = u64::try_from(observations.len())
162            .map_err(|_| ExtractorError::InvalidInput("too many observations".to_owned()))?;
163        let payload = serialize_bounded(&observations, &tracker)?;
164        tracker.check_structured_time()?;
165        Ok(ExtractionBatch {
166            source: ArtifactFingerprint {
167                repo_id: input.file.repo_id.clone(),
168                checkout_id: input.file.checkout_id.clone(),
169                path: input.file.path.clone(),
170                extractor: self.id().to_owned(),
171                content_hash: fingerprint.content_hash,
172                size_bytes: fingerprint.size_bytes,
173            },
174            payload,
175            output_count,
176            report: ExtractionReport {
177                discovered_files: 1,
178                parsed_files: 1,
179                skipped_files: 0,
180                completeness,
181                warnings,
182                evidence_count: output_count,
183                extractor_version: self.version(),
184                elapsed_ms: elapsed_ms(started),
185            },
186        })
187    }
188
189    async fn fingerprint(
190        &self,
191        input: &ExtractInput<'_>,
192    ) -> Result<crate::ContentFingerprint, ExtractorError> {
193        fingerprint_with_budgets(input, self.id(), &self.budgets)
194    }
195}
196
197fn syntax_language(language: FocusedSourceLanguage) -> SourceSyntaxLanguage {
198    match language {
199        FocusedSourceLanguage::JavaScript => SourceSyntaxLanguage::JavaScript,
200        FocusedSourceLanguage::TypeScript => SourceSyntaxLanguage::TypeScript,
201        FocusedSourceLanguage::Rust => SourceSyntaxLanguage::Rust,
202        FocusedSourceLanguage::Python => SourceSyntaxLanguage::Python,
203        FocusedSourceLanguage::Go => SourceSyntaxLanguage::Go,
204        FocusedSourceLanguage::Java => SourceSyntaxLanguage::Java,
205    }
206}
207
208/// Preflights source tokens before focused parsers allocate owned observation fields.
209///
210/// # Errors
211///
212/// Returns [`crate::ExtractionLimitExceeded`] when work, value, accumulated-string, or time
213/// budgets are exceeded.
214#[doc(hidden)]
215pub fn precheck_focused_source_values(
216    source: &str,
217    language: SourceSyntaxLanguage,
218    tracker: &mut ExtractionTracker,
219) -> Result<(), crate::ExtractionLimitExceeded> {
220    let bytes = source.as_bytes();
221    let mut cursor = 0_usize;
222    let mut accumulated = 0_u64;
223    while cursor < bytes.len() {
224        if cursor.is_multiple_of(1_024) {
225            tracker.check_structured_time()?;
226        }
227        let byte = bytes[cursor];
228        if matches!(byte, b'"' | b'\'' | b'`')
229            && (byte != b'\''
230                || language != SourceSyntaxLanguage::Rust
231                || rust_char_literal_end(bytes, cursor).is_some())
232        {
233            tracker.charge_work(1)?;
234            let delimiter = byte;
235            cursor = cursor.saturating_add(1);
236            let start = cursor;
237            let mut escaped = false;
238            while cursor < bytes.len() {
239                let byte = bytes[cursor];
240                if escaped {
241                    escaped = false;
242                } else if byte == b'\\' {
243                    escaped = true;
244                } else if byte == delimiter {
245                    break;
246                }
247                cursor = cursor.saturating_add(1);
248                if cursor.is_multiple_of(1_024) {
249                    tracker.check_structured_time()?;
250                }
251            }
252            let observed = u64::try_from(cursor.saturating_sub(start)).unwrap_or(u64::MAX);
253            tracker.check_string_bytes(observed)?;
254            accumulated = accumulated.saturating_add(observed);
255            tracker.check_accumulated_string_bytes(accumulated)?;
256        } else if byte == b'_' || byte.is_ascii_alphabetic() {
257            tracker.charge_work(1)?;
258            let start = cursor;
259            cursor = cursor.saturating_add(1);
260            while cursor < bytes.len()
261                && (bytes[cursor] == b'_' || bytes[cursor].is_ascii_alphanumeric())
262            {
263                cursor = cursor.saturating_add(1);
264            }
265            let observed = u64::try_from(cursor.saturating_sub(start)).unwrap_or(u64::MAX);
266            tracker.check_identifier_bytes(observed)?;
267            accumulated = accumulated.saturating_add(observed);
268            tracker.check_accumulated_string_bytes(accumulated)?;
269            continue;
270        }
271        cursor = cursor.saturating_add(1);
272    }
273    Ok(())
274}
275
276fn rust_char_literal_end(bytes: &[u8], index: usize) -> Option<usize> {
277    let value = index.checked_add(1)?;
278    let closing = if bytes.get(value) == Some(&b'\\') {
279        match bytes.get(value + 1)? {
280            b'u' if bytes.get(value + 2) == Some(&b'{') => {
281                (value + 3..bytes.len()).find(|cursor| bytes[*cursor] == b'}')? + 1
282            }
283            b'x' => value.checked_add(4)?,
284            _ => value.checked_add(2)?,
285        }
286    } else {
287        let width = std::str::from_utf8(bytes.get(value..)?)
288            .ok()?
289            .chars()
290            .next()?
291            .len_utf8();
292        value.checked_add(width)?
293    };
294    (bytes.get(closing) == Some(&b'\'')).then_some(closing + 1)
295}
296
297/// Built-in `OpenAPI` Generator metadata extractor.
298pub struct GeneratedClientMetadataExtractor {
299    budgets: ExtractionBudgets,
300}
301
302impl GeneratedClientMetadataExtractor {
303    /// Creates an extractor with safe default budgets.
304    #[must_use]
305    pub fn new() -> Self {
306        Self::with_budgets(ExtractionBudgets::default())
307    }
308
309    /// Creates an extractor with explicit effective per-invocation budgets.
310    #[must_use]
311    pub const fn with_budgets(budgets: ExtractionBudgets) -> Self {
312        Self { budgets }
313    }
314}
315
316impl Default for GeneratedClientMetadataExtractor {
317    fn default() -> Self {
318        Self::new()
319    }
320}
321
322#[async_trait]
323impl BoundaryExtractor for GeneratedClientMetadataExtractor {
324    fn id(&self) -> &'static str {
325        "code-system-graph.http.generated-client"
326    }
327
328    fn version(&self) -> Version {
329        Version::new(1, 0, 0)
330    }
331
332    fn supports(&self, file: &FileDescriptor) -> bool {
333        generated_client_path_supported(&file.path.display)
334    }
335
336    async fn discover(
337        &self,
338        context: &DiscoverContext<'_>,
339    ) -> Result<Vec<DiscoveredInput>, ExtractorError> {
340        Ok(context
341            .files
342            .iter()
343            .filter(|file| self.supports(file))
344            .cloned()
345            .map(|file| DiscoveredInput { file })
346            .collect())
347    }
348
349    async fn extract(&self, input: &ExtractInput<'_>) -> Result<ExtractionBatch, ExtractorError> {
350        if !self.supports(input.file) {
351            return Err(ExtractorError::InvalidInput(format!(
352                "{} does not support `{}`",
353                self.id(),
354                input.file.path.display
355            )));
356        }
357        let started = Instant::now();
358        let mut tracker =
359            crate::ExtractionTracker::new(&input.file.path.display, self.id(), &self.budgets);
360        tracker.check_input_bytes(u64::try_from(input.content.len()).unwrap_or(u64::MAX))?;
361        let fingerprint = fingerprint_with_budgets(input, self.id(), &self.budgets)?;
362        let source = std::str::from_utf8(input.content)?;
363        let metadata =
364            extract_generated_client_metadata(&input.file.path.display, source, &mut tracker)
365                .map_err(|error| match error {
366                    crate::GeneratedClientError::LimitExceeded(limit) => {
367                        ExtractorError::LimitExceeded(limit)
368                    }
369                    error => ExtractorError::InvalidInput(error.to_string()),
370                })?;
371        let output_count = u64::try_from(metadata.len())
372            .map_err(|_| ExtractorError::InvalidInput("too many metadata facts".to_owned()))?;
373        let payload = serialize_bounded(&metadata, &tracker)?;
374        tracker.check_structured_time()?;
375        Ok(ExtractionBatch {
376            source: ArtifactFingerprint {
377                repo_id: input.file.repo_id.clone(),
378                checkout_id: input.file.checkout_id.clone(),
379                path: input.file.path.clone(),
380                extractor: self.id().to_owned(),
381                content_hash: fingerprint.content_hash,
382                size_bytes: fingerprint.size_bytes,
383            },
384            payload,
385            output_count,
386            report: ExtractionReport {
387                discovered_files: 1,
388                parsed_files: 1,
389                skipped_files: 0,
390                completeness: ExtractionCompleteness::Complete,
391                warnings: Vec::new(),
392                evidence_count: output_count,
393                extractor_version: self.version(),
394                elapsed_ms: elapsed_ms(started),
395            },
396        })
397    }
398
399    async fn fingerprint(
400        &self,
401        input: &ExtractInput<'_>,
402    ) -> Result<crate::ContentFingerprint, ExtractorError> {
403        fingerprint_with_budgets(input, self.id(), &self.budgets)
404    }
405}
406
407/// Built-in package-manifest extractor for the mandatory package ecosystems.
408pub struct PackageManifestExtractor {
409    budgets: ExtractionBudgets,
410}
411
412impl PackageManifestExtractor {
413    /// Creates an extractor with safe default budgets.
414    #[must_use]
415    pub fn new() -> Self {
416        Self::with_budgets(ExtractionBudgets::default())
417    }
418
419    /// Creates an extractor with explicit effective per-invocation budgets.
420    #[must_use]
421    pub const fn with_budgets(budgets: ExtractionBudgets) -> Self {
422        Self { budgets }
423    }
424}
425
426impl Default for PackageManifestExtractor {
427    fn default() -> Self {
428        Self::new()
429    }
430}
431
432#[async_trait]
433impl BoundaryExtractor for PackageManifestExtractor {
434    fn id(&self) -> &'static str {
435        "code-system-graph.packages"
436    }
437
438    fn version(&self) -> Version {
439        Version::new(1, 0, 0)
440    }
441
442    fn supports(&self, file: &FileDescriptor) -> bool {
443        package_path_supported(&file.path.display)
444    }
445
446    async fn discover(
447        &self,
448        context: &DiscoverContext<'_>,
449    ) -> Result<Vec<DiscoveredInput>, ExtractorError> {
450        Ok(context
451            .files
452            .iter()
453            .filter(|file| self.supports(file))
454            .cloned()
455            .map(|file| DiscoveredInput { file })
456            .collect())
457    }
458
459    async fn extract(&self, input: &ExtractInput<'_>) -> Result<ExtractionBatch, ExtractorError> {
460        if !self.supports(input.file) {
461            return Err(ExtractorError::InvalidInput(format!(
462                "{} does not support `{}`",
463                self.id(),
464                input.file.path.display
465            )));
466        }
467        let started = Instant::now();
468        let mut tracker =
469            crate::ExtractionTracker::new(&input.file.path.display, self.id(), &self.budgets);
470        tracker.check_input_bytes(u64::try_from(input.content.len()).unwrap_or(u64::MAX))?;
471        let fingerprint = fingerprint_with_budgets(input, self.id(), &self.budgets)?;
472        let source = std::str::from_utf8(input.content)?;
473        let manifest =
474            extract_package_manifest_with_tracker(&input.file.path.display, source, &mut tracker)
475                .map_err(|error| match error {
476                crate::PackageManifestError::LimitExceeded(limit) => {
477                    ExtractorError::LimitExceeded(limit)
478                }
479                error => ExtractorError::InvalidInput(error.to_string()),
480            })?;
481        let output_count = [
482            manifest.packages.len(),
483            manifest.dependencies.len(),
484            manifest.workspace_members.len(),
485            manifest.exports.len(),
486            manifest.features.len(),
487            manifest.lockfiles.len(),
488        ]
489        .into_iter()
490        .try_fold(0_u64, |total, count| {
491            u64::try_from(count)
492                .ok()
493                .and_then(|count| total.checked_add(count))
494        })
495        .ok_or_else(|| ExtractorError::InvalidInput("too many package facts".to_owned()))?;
496        let payload = serialize_bounded(&manifest, &tracker)?;
497        tracker.check_structured_time()?;
498        Ok(ExtractionBatch {
499            source: ArtifactFingerprint {
500                repo_id: input.file.repo_id.clone(),
501                checkout_id: input.file.checkout_id.clone(),
502                path: input.file.path.clone(),
503                extractor: self.id().to_owned(),
504                content_hash: fingerprint.content_hash,
505                size_bytes: fingerprint.size_bytes,
506            },
507            payload,
508            output_count,
509            report: ExtractionReport {
510                discovered_files: 1,
511                parsed_files: 1,
512                skipped_files: 0,
513                completeness: ExtractionCompleteness::Complete,
514                warnings: Vec::new(),
515                evidence_count: output_count,
516                extractor_version: self.version(),
517                elapsed_ms: elapsed_ms(started),
518            },
519        })
520    }
521
522    async fn fingerprint(
523        &self,
524        input: &ExtractInput<'_>,
525    ) -> Result<crate::ContentFingerprint, ExtractorError> {
526        fingerprint_with_budgets(input, self.id(), &self.budgets)
527    }
528}
529
530fn extractor_source_syntax_error(error: crate::SourceSyntaxError) -> ExtractorError {
531    match error {
532        crate::SourceSyntaxError::LimitExceeded(limit) => ExtractorError::LimitExceeded(limit),
533        error => ExtractorError::InvalidInput(error.to_string()),
534    }
535}
536
537fn fingerprint_with_budgets(
538    input: &ExtractInput<'_>,
539    extractor: &str,
540    budgets: &ExtractionBudgets,
541) -> Result<ContentFingerprint, ExtractorError> {
542    let observed = u64::try_from(input.content.len()).unwrap_or(u64::MAX);
543    ExtractionTracker::new(&input.file.path.display, extractor, budgets)
544        .check_input_bytes(observed)?;
545    Ok(ContentFingerprint {
546        content_hash: blake3::hash(input.content).to_hex().to_string(),
547        size_bytes: observed,
548    })
549}
550
551/// Charges every retained value in one focused source observation.
552///
553/// # Errors
554///
555/// Returns [`crate::ExtractionLimitExceeded`] before retaining values beyond effective budgets.
556#[doc(hidden)]
557pub fn charge_source_observation(
558    observation: &SourceObservation,
559    tracker: &mut ExtractionTracker,
560) -> Result<(), crate::ExtractionLimitExceeded> {
561    crate::source_http::charge_source_observation_values(observation, tracker)
562}
563
564fn serialize_bounded<T: Serialize>(
565    value: &T,
566    tracker: &ExtractionTracker,
567) -> Result<Vec<u8>, ExtractorError> {
568    let mut writer = tracker.bounded_json_writer();
569    if let Err(error) = serde_json::to_writer(&mut writer, value) {
570        if let Some(limit) = tracker.output_limit_error(&writer) {
571            return Err(ExtractorError::LimitExceeded(limit));
572        }
573        return Err(ExtractorError::InvalidOutput(error));
574    }
575    Ok(writer.into_inner())
576}
577
578fn package_path_supported(path: &str) -> bool {
579    let name = path.rsplit(['/', '\\']).next().unwrap_or(path);
580    matches!(
581        name,
582        "package.json"
583            | "package-lock.json"
584            | "npm-shrinkwrap.json"
585            | "pnpm-lock.yaml"
586            | "yarn.lock"
587            | "pyproject.toml"
588            | "Cargo.toml"
589            | "go.mod"
590            | "go.work"
591            | "pom.xml"
592            | "build.gradle"
593            | "build.gradle.kts"
594            | "packages.config"
595    ) || (name.starts_with("requirements")
596        && std::path::Path::new(name)
597            .extension()
598            .is_some_and(|extension| extension.eq_ignore_ascii_case("txt")))
599        || name.to_ascii_lowercase().ends_with(".csproj")
600}
601
602fn generated_client_path_supported(path: &str) -> bool {
603    let path = std::path::Path::new(path);
604    let Some(name) = path.file_name().and_then(std::ffi::OsStr::to_str) else {
605        return false;
606    };
607    name == "openapitools.json"
608        || (matches!(name, "FILES" | "VERSION")
609            && path
610                .parent()
611                .and_then(std::path::Path::file_name)
612                .is_some_and(|parent| parent == ".openapi-generator"))
613}
614
615fn elapsed_ms(started: Instant) -> u64 {
616    u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX)
617}
618
619#[cfg(test)]
620mod tests {
621    use code_system_graph_model::{CheckoutId, NativePath, NativePathEncoding, RepoId};
622
623    use super::{
624        BoundaryExtractor, ExtractInput, ExtractionBudgets, ExtractorError, FileDescriptor, FocusedSourceExtractor, FocusedSourceLanguage, GeneratedClientMetadataExtractor, PackageManifestExtractor, SourceSyntaxLanguage
625    };
626
627    fn file(path: &str) -> FileDescriptor {
628        FileDescriptor {
629            repo_id: RepoId::new("repo:test"),
630            checkout_id: CheckoutId::new("checkout:test"),
631            path: NativePath {
632                encoding: NativePathEncoding::Utf8,
633                bytes: path.as_bytes().to_vec(),
634                display: path.to_owned(),
635            },
636            size_bytes: 1,
637        }
638    }
639
640    #[tokio::test]
641    async fn source_extractor_should_emit_versioned_json_without_source_text() {
642        let extractor = FocusedSourceExtractor::new(FocusedSourceLanguage::Rust);
643        let file = file("src/routes.rs");
644        let source =
645            b"use axum::{Router, routing::get}; Router::new().route(\"/health\", get(health));";
646        let result = extractor
647            .extract(&ExtractInput {
648                file: &file,
649                content: source,
650            })
651            .await;
652
653        assert!(matches!(
654            result,
655            Ok(batch)
656                if batch.output_count == 1
657                    && !String::from_utf8_lossy(&batch.payload).contains("Router::new")
658        ));
659    }
660
661    #[tokio::test]
662    async fn rust_macro_bodies_should_not_fail_structural_validation() {
663        let extractor = FocusedSourceExtractor::new(FocusedSourceLanguage::Rust);
664        let file = file("src/macros.rs");
665        let sources: &[&[u8]] = &[
666            b"macro_rules! delegate { () => { fn run(path: &str) { worker::execute(path); } }; }",
667            b"generate! { fn run(path: &str) { worker::execute(path); } }",
668        ];
669        for source in sources {
670            let batch = extractor
671                .extract(&ExtractInput {
672                    file: &file,
673                    content: source,
674                })
675                .await
676                .expect("valid macro body should pass structural validation");
677            assert!(batch.output_count > 0);
678        }
679    }
680
681    #[tokio::test]
682    async fn source_extractor_should_preflight_work_and_values() {
683        let file = file("src/routes.rs");
684        let source =
685            b"use axum::{Router, routing::get}; Router::new().route(\"/health\", get(health));";
686        let cases = [
687            (
688                ExtractionBudgets {
689                    max_work_units_per_artifact: 1,
690                    ..ExtractionBudgets::default()
691                },
692                crate::ExtractionResource::WorkUnits,
693            ),
694            (
695                ExtractionBudgets {
696                    max_string_bytes_per_value: 3,
697                    ..ExtractionBudgets::default()
698                },
699                crate::ExtractionResource::StringBytesPerValue,
700            ),
701        ];
702
703        for (budgets, resource) in cases {
704            let extractor =
705                FocusedSourceExtractor::with_budgets(FocusedSourceLanguage::Rust, budgets);
706            let result = extractor
707                .extract(&ExtractInput {
708                    file: &file,
709                    content: source,
710                })
711                .await;
712            assert!(matches!(
713                result,
714                Err(ExtractorError::LimitExceeded(error)) if error.resource == resource
715            ));
716        }
717    }
718
719    #[test]
720    fn rust_preflight_should_not_treat_a_lifetime_as_one_large_char_literal() {
721        let mut source = "type View<'a> = &'a str;\n".to_owned();
722        source.push_str(&"x;".repeat(35_000));
723        let budgets = ExtractionBudgets::default();
724        let mut tracker =
725            crate::ExtractionTracker::new("src/view.rs", "code-system-graph.source.rust", &budgets);
726
727        assert!(
728            super::precheck_focused_source_values(
729                &source,
730                SourceSyntaxLanguage::Rust,
731                &mut tracker
732            )
733            .is_ok()
734        );
735    }
736
737    #[tokio::test]
738    async fn source_extractor_should_charge_each_observation_before_retention() {
739        let file = file("tests/test_api.py");
740        let source = b"import requests\ndef test_create_order():\n    requests.post(\"https://api.test/v1/orders\")\n";
741        let one = ExtractionBudgets {
742            max_observations_per_artifact: 1,
743            ..ExtractionBudgets::default()
744        };
745        let rejected = FocusedSourceExtractor::with_budgets(FocusedSourceLanguage::Python, one)
746            .extract(&ExtractInput {
747                file: &file,
748                content: source,
749            })
750            .await;
751        assert!(matches!(
752            rejected,
753            Err(ExtractorError::LimitExceeded(error))
754                if error.resource == crate::ExtractionResource::Observations
755                    && error.observed == 2
756                    && error.maximum == 1
757        ));
758
759        let exact = ExtractionBudgets {
760            max_observations_per_artifact: 2,
761            ..ExtractionBudgets::default()
762        };
763        let accepted = FocusedSourceExtractor::with_budgets(FocusedSourceLanguage::Python, exact)
764            .extract(&ExtractInput {
765                file: &file,
766                content: source,
767            })
768            .await;
769        assert!(matches!(accepted, Ok(batch) if batch.output_count == 2));
770    }
771
772    #[tokio::test]
773    async fn javascript_extractor_should_preserve_next_file_route_context() {
774        let extractor = FocusedSourceExtractor::new(FocusedSourceLanguage::JavaScript);
775        let file = file("frontend/src/app/api/logout/route.js");
776        let source = b"export async function GET() { return new Response(); }\n";
777        let batch = extractor
778            .extract(&ExtractInput {
779                file: &file,
780                content: source,
781            })
782            .await
783            .expect("Next.js route should extract");
784        let observations: Vec<crate::SourceObservation> =
785            serde_json::from_slice(&batch.payload).expect("valid observation payload");
786
787        assert!(observations.iter().any(|observation| {
788            observation.framework == crate::SourceFramework::NextJs
789                && observation.method.as_deref() == Some("GET")
790                && observation.path.as_deref() == Some("/api/logout")
791        }));
792    }
793
794    #[tokio::test]
795    async fn generated_client_extractor_should_emit_explicit_config_facts() {
796        let extractor = GeneratedClientMetadataExtractor::default();
797        let file = file("openapitools.json");
798        let result = extractor
799            .extract(&ExtractInput {
800                file: &file,
801                content: br#"{"generator-cli":{"generators":{"client":{"generatorName":"rust","inputSpec":"openapi.yaml"}}}}"#,
802            })
803            .await;
804
805        assert!(matches!(result, Ok(batch) if batch.output_count == 1));
806    }
807
808    #[tokio::test]
809    async fn package_extractor_should_omit_evidence_source_lines_from_payload() {
810        let extractor = PackageManifestExtractor::default();
811        let file = file("Cargo.toml");
812        let source = b"[package]\nname = \"api\"\nversion = \"1.0.0\"\n";
813        let result = extractor
814            .extract(&ExtractInput {
815                file: &file,
816                content: source,
817            })
818            .await;
819
820        assert!(matches!(
821            result,
822            Ok(batch)
823                if batch.output_count == 1
824                    && !String::from_utf8_lossy(&batch.payload).contains("name =")
825        ));
826    }
827
828    #[tokio::test]
829    async fn built_in_extractors_should_honor_explicit_effective_budgets() {
830        let budgets = ExtractionBudgets {
831            max_serialized_output_bytes_per_artifact: 1,
832            ..ExtractionBudgets::default()
833        };
834        let extractor = GeneratedClientMetadataExtractor::with_budgets(budgets);
835        let file = file("openapitools.json");
836        let result = extractor
837            .extract(&ExtractInput {
838                file: &file,
839                content: br#"{"generator-cli":{"generators":{"client":{"generatorName":"rust"}}}}"#,
840            })
841            .await;
842
843        assert!(matches!(
844            result,
845            Err(ExtractorError::LimitExceeded(error))
846                if error.resource
847                    == crate::ExtractionResource::SerializedOutputBytes
848                    && error.maximum == 1
849        ));
850    }
851}