code_system_graph_core/
extractor.rs

1use async_trait::async_trait;
2use code_system_graph_model::{
3    ArtifactFingerprint, CheckoutId, NativePath, RepoId, StoredExtractorBatch
4};
5use semver::Version;
6use serde::{Deserialize, Serialize};
7use thiserror::Error;
8
9/// Maximum source size accepted by focused boundary extractors.
10pub const MAX_EXTRACTOR_INPUT_BYTES: usize = 8 * 1024 * 1024;
11
12/// Repository file metadata available during extractor discovery.
13#[derive(Debug, Clone, PartialEq, Eq)]
14pub struct FileDescriptor {
15    /// Repository containing the file.
16    pub repo_id: RepoId,
17    /// Concrete checkout containing the file.
18    pub checkout_id: CheckoutId,
19    /// Lossless repository-relative path.
20    pub path: NativePath,
21    /// Exact source size in bytes.
22    pub size_bytes: u64,
23}
24
25/// Bounded repository inventory supplied to an extractor.
26#[derive(Debug, Clone)]
27pub struct DiscoverContext<'a> {
28    /// Candidate files in deterministic path order.
29    pub files: &'a [FileDescriptor],
30}
31
32/// One file selected by an extractor.
33#[derive(Debug, Clone, PartialEq, Eq)]
34pub struct DiscoveredInput {
35    /// Selected file metadata.
36    pub file: FileDescriptor,
37}
38
39/// Immutable source input supplied to a focused extractor.
40#[derive(Debug, Clone)]
41pub struct ExtractInput<'a> {
42    /// Selected file metadata.
43    pub file: &'a FileDescriptor,
44    /// Bounded source content.
45    pub content: &'a [u8],
46}
47
48/// Content identity returned independently from extraction.
49#[derive(Debug, Clone, PartialEq, Eq)]
50pub struct ContentFingerprint {
51    /// BLAKE3 content hash.
52    pub content_hash: String,
53    /// Exact source size in bytes.
54    pub size_bytes: u64,
55}
56
57/// Explicit completeness of one focused extraction.
58#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
59#[serde(rename_all = "snake_case")]
60pub enum ExtractionCompleteness {
61    /// Every supported construct in the input was parsed.
62    Complete,
63    /// Unsupported or dynamic constructs were observed and reported.
64    Partial,
65}
66
67/// Audit metrics and diagnostics for one source-owned batch.
68#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
69pub struct ExtractionReport {
70    /// Number of files selected by this batch.
71    pub discovered_files: u64,
72    /// Number of files parsed successfully.
73    pub parsed_files: u64,
74    /// Number of files skipped.
75    pub skipped_files: u64,
76    /// Explicit result completeness.
77    pub completeness: ExtractionCompleteness,
78    /// Bounded warnings; these must not contain source contents or secrets.
79    pub warnings: Vec<String>,
80    /// Number of evidence records emitted into the payload.
81    pub evidence_count: u64,
82    /// Extractor semantic version.
83    pub extractor_version: Version,
84    /// Bounded elapsed wall-clock time.
85    pub elapsed_ms: u64,
86}
87
88/// Versioned, source-owned output produced by a boundary extractor.
89#[derive(Debug, Clone, PartialEq, Eq)]
90pub struct ExtractionBatch {
91    /// Source fingerprint that owns and invalidates this output.
92    pub source: ArtifactFingerprint,
93    /// Deterministic UTF-8 JSON observations without source text.
94    pub payload: Vec<u8>,
95    /// Number of observations encoded in the payload.
96    pub output_count: u64,
97    /// Extraction diagnostics.
98    pub report: ExtractionReport,
99}
100
101impl ExtractionBatch {
102    /// Converts the batch to its persistence representation.
103    #[must_use]
104    pub fn into_stored(
105        self,
106        budget_fingerprint: String,
107        source_was_lossy: bool,
108    ) -> StoredExtractorBatch {
109        StoredExtractorBatch {
110            source: self.source,
111            extractor_version: self.report.extractor_version.to_string(),
112            budget_fingerprint,
113            source_was_lossy,
114            output_count: self.output_count,
115            payload: self.payload,
116        }
117    }
118}
119
120/// Failure returned by focused boundary extractors.
121#[derive(Debug, Error)]
122pub enum ExtractorError {
123    /// Extraction exceeded one configured invocation resource.
124    #[error(transparent)]
125    LimitExceeded(#[from] crate::ExtractionLimitExceeded),
126    /// Input exceeds the documented extraction budget.
127    #[error("extractor input is {actual} bytes; maximum is {maximum}")]
128    InputTooLarge {
129        /// Observed byte count.
130        actual: usize,
131        /// Configured maximum.
132        maximum: usize,
133    },
134    /// Source bytes are not valid for a text extractor.
135    #[error("extractor input is not valid UTF-8")]
136    InvalidUtf8(#[from] std::str::Utf8Error),
137    /// Structured output could not be encoded.
138    #[error("extractor output could not be encoded: {0}")]
139    InvalidOutput(#[from] serde_json::Error),
140    /// Extractor-specific failure with a bounded, non-sensitive explanation.
141    #[error("{0}")]
142    InvalidInput(String),
143}
144
145/// Focused, deterministic contract extractor.
146#[async_trait]
147pub trait BoundaryExtractor: Send + Sync {
148    /// Stable extractor identity.
149    fn id(&self) -> &'static str;
150
151    /// Extractor and payload-schema semantic version.
152    fn version(&self) -> Version;
153
154    /// Returns whether this extractor can consume a candidate file.
155    fn supports(&self, file: &FileDescriptor) -> bool;
156
157    /// Selects supported files from a bounded deterministic inventory.
158    async fn discover(
159        &self,
160        context: &DiscoverContext<'_>,
161    ) -> Result<Vec<DiscoveredInput>, ExtractorError>;
162
163    /// Produces a complete replacement batch for one source file.
164    async fn extract(&self, input: &ExtractInput<'_>) -> Result<ExtractionBatch, ExtractorError>;
165
166    /// Computes the source fingerprint used for incremental reuse.
167    async fn fingerprint(
168        &self,
169        input: &ExtractInput<'_>,
170    ) -> Result<ContentFingerprint, ExtractorError>;
171}
172
173/// Computes a bounded source fingerprint shared by focused extractors.
174///
175/// # Errors
176///
177/// Returns [`ExtractorError::InputTooLarge`] when the input exceeds the extraction budget.
178pub fn fingerprint_content(content: &[u8]) -> Result<ContentFingerprint, ExtractorError> {
179    if content.len() > MAX_EXTRACTOR_INPUT_BYTES {
180        return Err(ExtractorError::InputTooLarge {
181            actual: content.len(),
182            maximum: MAX_EXTRACTOR_INPUT_BYTES,
183        });
184    }
185    Ok(ContentFingerprint {
186        content_hash: blake3::hash(content).to_hex().to_string(),
187        size_bytes: u64::try_from(content.len()).map_err(|_| {
188            ExtractorError::InvalidInput("source size exceeds the supported range".to_owned())
189        })?,
190    })
191}
192
193#[cfg(test)]
194mod tests {
195    use super::{ExtractorError, MAX_EXTRACTOR_INPUT_BYTES, fingerprint_content};
196
197    #[test]
198    fn fingerprint_should_be_deterministic_and_bounded() {
199        let first = fingerprint_content(b"GET /orders");
200        let second = fingerprint_content(b"GET /orders");
201
202        assert!(matches!(
203            (first, second),
204            (Ok(left), Ok(right)) if left == right
205        ));
206        assert!(matches!(
207            fingerprint_content(&vec![0; MAX_EXTRACTOR_INPUT_BYTES + 1]),
208            Err(ExtractorError::InputTooLarge { .. })
209        ));
210    }
211}
code_system_graph_core/extractor.rs

code_system_graph_core/
extractor.rs
