1use async_trait::async_trait;
2use code_system_graph_model::{
3 ArtifactFingerprint, CheckoutId, NativePath, RepoId, StoredExtractorBatch
4};
5use semver::Version;
6use serde::{Deserialize, Serialize};
7use thiserror::Error;
8
9pub const MAX_EXTRACTOR_INPUT_BYTES: usize = 8 * 1024 * 1024;
11
12#[derive(Debug, Clone, PartialEq, Eq)]
14pub struct FileDescriptor {
15 pub repo_id: RepoId,
17 pub checkout_id: CheckoutId,
19 pub path: NativePath,
21 pub size_bytes: u64,
23}
24
25#[derive(Debug, Clone)]
27pub struct DiscoverContext<'a> {
28 pub files: &'a [FileDescriptor],
30}
31
32#[derive(Debug, Clone, PartialEq, Eq)]
34pub struct DiscoveredInput {
35 pub file: FileDescriptor,
37}
38
39#[derive(Debug, Clone)]
41pub struct ExtractInput<'a> {
42 pub file: &'a FileDescriptor,
44 pub content: &'a [u8],
46}
47
48#[derive(Debug, Clone, PartialEq, Eq)]
50pub struct ContentFingerprint {
51 pub content_hash: String,
53 pub size_bytes: u64,
55}
56
57#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
59#[serde(rename_all = "snake_case")]
60pub enum ExtractionCompleteness {
61 Complete,
63 Partial,
65}
66
67#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
69pub struct ExtractionReport {
70 pub discovered_files: u64,
72 pub parsed_files: u64,
74 pub skipped_files: u64,
76 pub completeness: ExtractionCompleteness,
78 pub warnings: Vec<String>,
80 pub evidence_count: u64,
82 pub extractor_version: Version,
84 pub elapsed_ms: u64,
86}
87
88#[derive(Debug, Clone, PartialEq, Eq)]
90pub struct ExtractionBatch {
91 pub source: ArtifactFingerprint,
93 pub payload: Vec<u8>,
95 pub output_count: u64,
97 pub report: ExtractionReport,
99}
100
101impl ExtractionBatch {
102 #[must_use]
104 pub fn into_stored(
105 self,
106 budget_fingerprint: String,
107 source_was_lossy: bool,
108 ) -> StoredExtractorBatch {
109 StoredExtractorBatch {
110 source: self.source,
111 extractor_version: self.report.extractor_version.to_string(),
112 budget_fingerprint,
113 source_was_lossy,
114 output_count: self.output_count,
115 payload: self.payload,
116 }
117 }
118}
119
120#[derive(Debug, Error)]
122pub enum ExtractorError {
123 #[error(transparent)]
125 LimitExceeded(#[from] crate::ExtractionLimitExceeded),
126 #[error("extractor input is {actual} bytes; maximum is {maximum}")]
128 InputTooLarge {
129 actual: usize,
131 maximum: usize,
133 },
134 #[error("extractor input is not valid UTF-8")]
136 InvalidUtf8(#[from] std::str::Utf8Error),
137 #[error("extractor output could not be encoded: {0}")]
139 InvalidOutput(#[from] serde_json::Error),
140 #[error("{0}")]
142 InvalidInput(String),
143}
144
145#[allow(clippy::double_must_use)]
147#[async_trait]
148pub trait BoundaryExtractor: Send + Sync {
149 fn id(&self) -> &'static str;
151
152 fn version(&self) -> Version;
154
155 fn supports(&self, file: &FileDescriptor) -> bool;
157
158 async fn discover(
160 &self,
161 context: &DiscoverContext<'_>,
162 ) -> Result<Vec<DiscoveredInput>, ExtractorError>;
163
164 async fn extract(&self, input: &ExtractInput<'_>) -> Result<ExtractionBatch, ExtractorError>;
166
167 async fn fingerprint(
169 &self,
170 input: &ExtractInput<'_>,
171 ) -> Result<ContentFingerprint, ExtractorError>;
172}
173
174pub fn fingerprint_content(content: &[u8]) -> Result<ContentFingerprint, ExtractorError> {
180 if content.len() > MAX_EXTRACTOR_INPUT_BYTES {
181 return Err(ExtractorError::InputTooLarge {
182 actual: content.len(),
183 maximum: MAX_EXTRACTOR_INPUT_BYTES,
184 });
185 }
186 Ok(ContentFingerprint {
187 content_hash: blake3::hash(content).to_hex().to_string(),
188 size_bytes: u64::try_from(content.len()).map_err(|_| {
189 ExtractorError::InvalidInput("source size exceeds the supported range".to_owned())
190 })?,
191 })
192}
193
194#[cfg(test)]
195mod tests {
196 use super::{ExtractorError, MAX_EXTRACTOR_INPUT_BYTES, fingerprint_content};
197
198 #[test]
199 fn fingerprint_should_be_deterministic_and_bounded() {
200 let first = fingerprint_content(b"GET /orders");
201 let second = fingerprint_content(b"GET /orders");
202
203 assert!(matches!(
204 (first, second),
205 (Ok(left), Ok(right)) if left == right
206 ));
207 assert!(matches!(
208 fingerprint_content(&vec![0; MAX_EXTRACTOR_INPUT_BYTES + 1]),
209 Err(ExtractorError::InputTooLarge { .. })
210 ));
211 }
212}