Skip to main content

code_system_graph_core/
extractor.rs

1use async_trait::async_trait;
2use code_system_graph_model::{
3    ArtifactFingerprint, CheckoutId, NativePath, RepoId, StoredExtractorBatch
4};
5use semver::Version;
6use serde::{Deserialize, Serialize};
7use thiserror::Error;
8
9/// Maximum source size accepted by focused boundary extractors.
10pub const MAX_EXTRACTOR_INPUT_BYTES: usize = 8 * 1024 * 1024;
11
12/// Repository file metadata available during extractor discovery.
13#[derive(Debug, Clone, PartialEq, Eq)]
14pub struct FileDescriptor {
15    /// Repository containing the file.
16    pub repo_id: RepoId,
17    /// Concrete checkout containing the file.
18    pub checkout_id: CheckoutId,
19    /// Lossless repository-relative path.
20    pub path: NativePath,
21    /// Exact source size in bytes.
22    pub size_bytes: u64,
23}
24
25/// Bounded repository inventory supplied to an extractor.
26#[derive(Debug, Clone)]
27pub struct DiscoverContext<'a> {
28    /// Candidate files in deterministic path order.
29    pub files: &'a [FileDescriptor],
30}
31
32/// One file selected by an extractor.
33#[derive(Debug, Clone, PartialEq, Eq)]
34pub struct DiscoveredInput {
35    /// Selected file metadata.
36    pub file: FileDescriptor,
37}
38
39/// Immutable source input supplied to a focused extractor.
40#[derive(Debug, Clone)]
41pub struct ExtractInput<'a> {
42    /// Selected file metadata.
43    pub file: &'a FileDescriptor,
44    /// Bounded source content.
45    pub content: &'a [u8],
46}
47
48/// Content identity returned independently from extraction.
49#[derive(Debug, Clone, PartialEq, Eq)]
50pub struct ContentFingerprint {
51    /// BLAKE3 content hash.
52    pub content_hash: String,
53    /// Exact source size in bytes.
54    pub size_bytes: u64,
55}
56
57/// Explicit completeness of one focused extraction.
58#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
59#[serde(rename_all = "snake_case")]
60pub enum ExtractionCompleteness {
61    /// Every supported construct in the input was parsed.
62    Complete,
63    /// Unsupported or dynamic constructs were observed and reported.
64    Partial,
65}
66
67/// Audit metrics and diagnostics for one source-owned batch.
68#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
69pub struct ExtractionReport {
70    /// Number of files selected by this batch.
71    pub discovered_files: u64,
72    /// Number of files parsed successfully.
73    pub parsed_files: u64,
74    /// Number of files skipped.
75    pub skipped_files: u64,
76    /// Explicit result completeness.
77    pub completeness: ExtractionCompleteness,
78    /// Bounded warnings; these must not contain source contents or secrets.
79    pub warnings: Vec<String>,
80    /// Number of evidence records emitted into the payload.
81    pub evidence_count: u64,
82    /// Extractor semantic version.
83    pub extractor_version: Version,
84    /// Bounded elapsed wall-clock time.
85    pub elapsed_ms: u64,
86}
87
88/// Versioned, source-owned output produced by a boundary extractor.
89#[derive(Debug, Clone, PartialEq, Eq)]
90pub struct ExtractionBatch {
91    /// Source fingerprint that owns and invalidates this output.
92    pub source: ArtifactFingerprint,
93    /// Deterministic UTF-8 JSON observations without source text.
94    pub payload: Vec<u8>,
95    /// Number of observations encoded in the payload.
96    pub output_count: u64,
97    /// Extraction diagnostics.
98    pub report: ExtractionReport,
99}
100
101impl ExtractionBatch {
102    /// Converts the batch to its persistence representation.
103    #[must_use]
104    pub fn into_stored(
105        self,
106        budget_fingerprint: String,
107        source_was_lossy: bool,
108    ) -> StoredExtractorBatch {
109        StoredExtractorBatch {
110            source: self.source,
111            extractor_version: self.report.extractor_version.to_string(),
112            budget_fingerprint,
113            source_was_lossy,
114            output_count: self.output_count,
115            payload: self.payload,
116        }
117    }
118}
119
120/// Failure returned by focused boundary extractors.
121#[derive(Debug, Error)]
122pub enum ExtractorError {
123    /// Extraction exceeded one configured invocation resource.
124    #[error(transparent)]
125    LimitExceeded(#[from] crate::ExtractionLimitExceeded),
126    /// Input exceeds the documented extraction budget.
127    #[error("extractor input is {actual} bytes; maximum is {maximum}")]
128    InputTooLarge {
129        /// Observed byte count.
130        actual: usize,
131        /// Configured maximum.
132        maximum: usize,
133    },
134    /// Source bytes are not valid for a text extractor.
135    #[error("extractor input is not valid UTF-8")]
136    InvalidUtf8(#[from] std::str::Utf8Error),
137    /// Structured output could not be encoded.
138    #[error("extractor output could not be encoded: {0}")]
139    InvalidOutput(#[from] serde_json::Error),
140    /// Extractor-specific failure with a bounded, non-sensitive explanation.
141    #[error("{0}")]
142    InvalidInput(String),
143}
144
145/// Focused, deterministic contract extractor.
146#[allow(clippy::double_must_use)]
147#[async_trait]
148pub trait BoundaryExtractor: Send + Sync {
149    /// Stable extractor identity.
150    fn id(&self) -> &'static str;
151
152    /// Extractor and payload-schema semantic version.
153    fn version(&self) -> Version;
154
155    /// Returns whether this extractor can consume a candidate file.
156    fn supports(&self, file: &FileDescriptor) -> bool;
157
158    /// Selects supported files from a bounded deterministic inventory.
159    async fn discover(
160        &self,
161        context: &DiscoverContext<'_>,
162    ) -> Result<Vec<DiscoveredInput>, ExtractorError>;
163
164    /// Produces a complete replacement batch for one source file.
165    async fn extract(&self, input: &ExtractInput<'_>) -> Result<ExtractionBatch, ExtractorError>;
166
167    /// Computes the source fingerprint used for incremental reuse.
168    async fn fingerprint(
169        &self,
170        input: &ExtractInput<'_>,
171    ) -> Result<ContentFingerprint, ExtractorError>;
172}
173
174/// Computes a bounded source fingerprint shared by focused extractors.
175///
176/// # Errors
177///
178/// Returns [`ExtractorError::InputTooLarge`] when the input exceeds the extraction budget.
179pub fn fingerprint_content(content: &[u8]) -> Result<ContentFingerprint, ExtractorError> {
180    if content.len() > MAX_EXTRACTOR_INPUT_BYTES {
181        return Err(ExtractorError::InputTooLarge {
182            actual: content.len(),
183            maximum: MAX_EXTRACTOR_INPUT_BYTES,
184        });
185    }
186    Ok(ContentFingerprint {
187        content_hash: blake3::hash(content).to_hex().to_string(),
188        size_bytes: u64::try_from(content.len()).map_err(|_| {
189            ExtractorError::InvalidInput("source size exceeds the supported range".to_owned())
190        })?,
191    })
192}
193
194#[cfg(test)]
195mod tests {
196    use super::{ExtractorError, MAX_EXTRACTOR_INPUT_BYTES, fingerprint_content};
197
198    #[test]
199    fn fingerprint_should_be_deterministic_and_bounded() {
200        let first = fingerprint_content(b"GET /orders");
201        let second = fingerprint_content(b"GET /orders");
202
203        assert!(matches!(
204            (first, second),
205            (Ok(left), Ok(right)) if left == right
206        ));
207        assert!(matches!(
208            fingerprint_content(&vec![0; MAX_EXTRACTOR_INPUT_BYTES + 1]),
209            Err(ExtractorError::InputTooLarge { .. })
210        ));
211    }
212}